From fc705a2db3fec2a37cd8c3a35da140133a9a9e82 Mon Sep 17 00:00:00 2001 From: Beth Skurrie Date: Fri, 30 Mar 2018 20:04:32 +1100 Subject: [PATCH] feat: add rake task to clean unused data to improve performance Currently, it deletes more verifications than expected, but if you are building the provider often, and you are not using can-i-deploy, this should be ok. https://github.com/pact-foundation/pact_broker/issues/203 --- lib/pact_broker/db/clean.rb | 49 +++++++++++++++++++++ lib/pact_broker/tasks.rb | 1 + lib/pact_broker/tasks/clean_task.rb | 25 +++++++++++ spec/lib/pact_broker/db/clean_spec.rb | 62 +++++++++++++++++++++++++++ 4 files changed, 137 insertions(+) create mode 100644 lib/pact_broker/db/clean.rb create mode 100644 lib/pact_broker/tasks/clean_task.rb create mode 100644 spec/lib/pact_broker/db/clean_spec.rb diff --git a/lib/pact_broker/db/clean.rb b/lib/pact_broker/db/clean.rb new file mode 100644 index 000000000..dfc8e1316 --- /dev/null +++ b/lib/pact_broker/db/clean.rb @@ -0,0 +1,49 @@ +require 'sequel' +require 'pact_broker/project_root' + +module PactBroker + module DB + class Clean + def self.call database_connection, options = {} + new(database_connection, options).call + end + + def initialize database_connection, options = {} + @db = database_connection + @options = options + end + + def call + db[:verifications].where(id: db[:materialized_head_matrix].select(:verification_id)).invert.delete + pp_ids = db[:materialized_head_matrix].select(:pact_publication_id) + + triggered_webhook_ids = db[:triggered_webhooks].where(pact_publication_id: pp_ids).invert.select(:id) + db[:webhook_executions].where(triggered_webhook_id: triggered_webhook_ids).delete + db[:triggered_webhooks].where(id: triggered_webhook_ids).delete + db[:webhook_executions].where(pact_publication_id: pp_ids).invert.delete + + db[:pact_publications].where(id: pp_ids).invert.delete + + referenced_pact_version_ids = db[:pact_publications].select(:pact_version_id).collect{ | h| h[:pact_version_id] } + + db[:verifications].select(:pact_version_id).collect{ | h| h[:pact_version_id] } + db[:pact_versions].where(id: referenced_pact_version_ids).invert.delete + + referenced_version_ids = db[:pact_publications].select(:consumer_version_id).collect{ | h| h[:consumer_version_id] } + + db[:verifications].select(:provider_version_id).collect{ | h| h[:provider_version_id] } + + db[:tags].where(version_id: referenced_version_ids).invert.delete + db[:versions].where(id: referenced_version_ids).invert.delete + + db[:materialized_matrix].delete + db[:materialized_matrix].insert(db[:matrix].select_all) + db[:materialized_head_matrix].delete + db[:materialized_head_matrix].insert(db[:head_matrix].select_all) + end + + private + + attr_reader :db + + end + end +end diff --git a/lib/pact_broker/tasks.rb b/lib/pact_broker/tasks.rb index 24741e128..42b6e608e 100644 --- a/lib/pact_broker/tasks.rb +++ b/lib/pact_broker/tasks.rb @@ -1,2 +1,3 @@ require 'pact_broker/tasks/migration_task' require 'pact_broker/tasks/version_task' +require 'pact_broker/tasks/clean_task' diff --git a/lib/pact_broker/tasks/clean_task.rb b/lib/pact_broker/tasks/clean_task.rb new file mode 100644 index 000000000..d93639d99 --- /dev/null +++ b/lib/pact_broker/tasks/clean_task.rb @@ -0,0 +1,25 @@ +module PactBroker + module DB + class CleanTask < ::Rake::TaskLib + + attr_accessor :database_connection + + def initialize &block + rake_task &block + end + + def rake_task &block + namespace :pact_broker do + namespace :db do + desc "Clean unused pacts and verifications from database" + task :clean do | t, args | + require 'pact_broker/db/clean' + instance_eval(&block) + PactBroker::DB::Clean.call(database_connection) + end + end + end + end + end + end +end \ No newline at end of file diff --git a/spec/lib/pact_broker/db/clean_spec.rb b/spec/lib/pact_broker/db/clean_spec.rb new file mode 100644 index 000000000..000adab05 --- /dev/null +++ b/spec/lib/pact_broker/db/clean_spec.rb @@ -0,0 +1,62 @@ +require 'pact_broker/db/clean' + +module PactBroker + module DB + describe Clean do + describe ".call" do + + let(:td) { TestDataBuilder.new } + before do + td.create_pact_with_hierarchy("Foo", "0", "Bar") + .create_consumer_version_tag("prod") + .create_consumer_version("1") + .create_pact + .create_consumer_version_tag("prod") + .comment("keep") + .create_verification(provider_version: "20") + .create_consumer_version("2") + .create_pact + .comment("don't keep") + .create_webhook + .create_triggered_webhook + .create_webhook_execution + .create_deprecated_webhook_execution + .create_verification(provider_version: "30") + .create_consumer_version("3") + .create_pact + .comment("keep") + .create_verification(provider_version: "40") + .create_verification(provider_version: "50", number: 2) + end + + subject { Clean.call(PactBroker::DB.connection) } + let(:db) { PactBroker::DB.connection } + + it "does not delete any rows in the head matrix" do + head_matrix_before = db[:head_matrix].select_all + subject + head_matrix_after = db[:head_matrix].select_all + expect(head_matrix_before).to eq head_matrix_after + end + + it "deletes rows that aren't the latest or latest tagged" do + subject + expect(db[:materialized_matrix].where(consumer_version_number: "2").count).to eq 0 + end + + it "deletes orphan pact_versions" do + subject + expect(db[:pact_versions].count).to eq 2 + end + + it "deletes orphan versions" do + subject + expect(db[:versions].where(number: "20").count).to be 1 + expect(db[:versions].where(number: "30").count).to be 0 + expect(db[:versions].where(number: "40").count).to be 0 + expect(db[:versions].where(number: "50").count).to be 1 + end + end + end + end +end