Merge pull request #958 from thecartercenter/12476_add_manual_cloning

12476: add back in manual cloning

app/models/cloning/exporter.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require "fileutils"
+
+# Note: Documented in "Server to Server Copy" wiki page.
+# TODO: This class needs tests.
+module Cloning
+  # Outputs data to CSV ZIP bundle.
+  class Exporter
+    attr_accessor :relations, :options
+
+    def initialize(relations, **options)
+      self.relations = relations
+      self.options = options
+    end
+
+    def export
+      expander = RelationExpander.new(relations, dont_implicitly_expand: options[:dont_implicitly_expand])
+      buffer = Zip::OutputStream.write_buffer do |out|
+        expander.expanded.each do |klass, relations|
+          # TODO: Improve this logic a bit, make it more structured and check table name
+          col_names = klass.column_names - %w[standard_copy last_mission_id]
+          relations.each_with_index do |relation, idx|
+            out.put_next_entry("#{klass.name.tr(':', '_')}-#{idx}.csv")
+            relation = relation.select(col_names.join(", ")) unless col_names == klass.column_names
+            relation.copy_to { |line| out.write(line) }
+          end
+        end
+      end
+      FileUtils.mkdir_p(export_dir)
+      File.open(zipfile_path, "wb") { |f| f.write(buffer.string) }
+    end
+
+    private
+
+    def export_dir
+      @export_dir ||= Rails.root.join("tmp/exports")
+    end
+
+    def zipfile_path
+      @zipfile_path ||= export_dir.join("#{Time.zone.now.to_s(:filename_datetime)}.zip")
+    end
+  end
+end

app/models/cloning/importer.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+# Note: Documented in "Server to Server Copy" wiki page.
+# TODO: This class needs tests.
+module Cloning
+  # Imports from CSV ZIP bundle.
+  class Importer
+    attr_accessor :zip_file
+
+    def initialize(zip_file)
+      self.zip_file = zip_file
+    end
+
+    def import
+      ApplicationRecord.transaction do
+        # Defer constraints so that constraints are not checked until all data is loaded.
+        SqlRunner.instance.run("SET CONSTRAINTS ALL DEFERRED")
+        Zip::InputStream.open(zip_file) do |io|
+          index = 0
+          while (entry = io.get_next_entry)
+            class_name = entry.name.match(/\A(\w+)-\d+\.csv\z/)[1]
+            klass = class_name.sub(/.csv$/, "").tr("_", ":").constantize
+            tmp_table = "tmp_table_#{index}"
+            SqlRunner.instance.run("CREATE TEMP TABLE #{tmp_table}
+              ON COMMIT DROP AS SELECT * FROM #{klass.table_name} WITH NO DATA")
+            klass.copy_from(io, table: tmp_table)
+            col_names = klass.column_names - %w[standard_copy last_mission_id]
+            select = col_names == klass.column_names ? "*" : col_names.join(", ")
+            insert_cols = col_names == klass.column_names ? "" : "(#{col_names.join(', ')})"
+            SqlRunner.instance.run("INSERT INTO #{klass.table_name}#{insert_cols}
+              SELECT #{select} FROM #{tmp_table} ON CONFLICT DO NOTHING")
+            index += 1
+          end
+        end
+      end
+    end
+  end
+end

app/models/cloning/relation_expander.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+# Note: Documented in "Server to Server Copy" wiki page.
+# TODO: This class needs tests.
+module Cloning
+  # Expands a given set of relations to include all necessary related objects.
+  class RelationExpander
+    attr_accessor :initial_relations, :relations_by_class, :options
+
+    def initialize(relations, **options)
+      self.options = options
+      options[:dont_implicitly_expand] ||= []
+      self.initial_relations = relations
+      self.relations_by_class = relations.group_by(&:klass)
+    end
+
+    # Returns a hash of form {ModelClass => [Relation, Relation, ...], ...}, mapping model classes
+    # to arrays of Relations.
+    def expanded
+      initial_relations.each { |r| expand(r) }
+      relations_by_class
+    end
+
+    private
+
+    def expand(relation)
+      (relation.klass.clone_options[:follow] || []).each do |assn_name|
+        assn = relation.klass.reflect_on_association(assn_name)
+
+        # dont_implicitly_expand is provided if the caller wants to indicate that one of the initial_relations
+        # should cover all relevant rows and therefore implicit expansion is not necessary. This improves
+        # performance by simplifying the eventual SQL queries.
+        next if options[:dont_implicitly_expand].include?(assn.klass)
+
+        new_rel = if assn.belongs_to?
+                    assn.klass.where("id IN (#{relation.select(assn.foreign_key).to_sql})")
+                  else
+                    assn.klass.where("#{assn.foreign_key} IN (#{relation.select(:id).to_sql})")
+                  end
+        (relations_by_class[assn.klass] ||= []) << new_rel
+        expand(new_rel)
+      end
+    end
+  end
+end