base people importer on csv

app/domain/sac_imports/csv_source.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+class SacImports::CsvSource
+  NIL_VALUES = ["", "NULL", "null", "Null"].freeze
+  SOURCE_HEADERS = {
+    NAV1: {
+      navision_id: "No_",
+      navision_name: "Name",
+      navision_membership_years: "Vereinsmitgliederjahre",
+      first_name: "First Name",
+      last_name: "Surname",
+      address_care_of: "Name 2",
+      postbox: "Address 2",
+      address: "Address",
+      street: "Street Name",
+      housenumber: "Street No_",
+      country: "Country_Region Code",
+      town: "City",
+      zip_code: "Post Code",
+      email: "E-Mail",
+      phone_private: "Phone No_",
+      phone_mobile: "Mobile Phone No_",
+      phone_fax: "Fax No_",
+      birthday: "Date of Birth",
+      gender: "Geschlecht",
+      language: "Language Code",
+      sac_remark_section_1: "Sektionsinfo 1 Bemerkung",
+      sac_remark_section_2: "Sektionsinfo 2 Bemerkung",
+      sac_remark_section_3: "Sektionsinfo 3 Bemerkung",
+      sac_remark_section_4: "Sektionsinfo 4 Bemerkung",
+      sac_remark_section_5: "Sektionsinfo 5 Bemerkung",
+      sac_remark_national_office: "Geschäftsstelle Bemerkung"
+    },
+    NAV2: {
+      navision_id: "Mitgliedernummer",
+      household_key: "Familien-Nr.",
+      group_navision_id: "Sektion",
+      navision_name: "Name",
+      navision_membership_years: "Vereinsmitgliederjahre"
+    }
+    # NAV3: {},
+    # WSO21: {},
+    # WSO22: {}
+  }.freeze
+
+  AVAILABLE_SOURCES = SOURCE_HEADERS.keys.freeze
+
+  def initialize(source_name)
+    @source_name = source_name
+    assert_available_source
+  end
+
+  def rows
+    data = []
+    CSV.foreach(path, headers: true) do |row|
+      data << process_row(row)
+    end
+    data
+  end
+
+  private
+
+  def process_row(row)
+    row = row.to_h
+    hash = {}
+    headers.keys.each do |header_key|
+      value = row[headers[header_key]]
+      value = nil if NIL_VALUES.include?(value)
+
+      hash[header_key] = value
+    end
+    hash
+  end
+
+  def path
+    files = Dir.glob("#{source_dir}/#{@source_name}_*.csv")
+    raise("No source file #{@source_name}_*.csv found in #{source_dir}.") if files.empty?
+
+    source_dir.join(files.first)
+  end
+
+  def headers
+    SOURCE_HEADERS[@source_name]
+  end
+
+  def source_dir
+    Rails.root.join("tmp", "sac_imports_src")
+  end
+
+  def assert_available_source
+    unless AVAILABLE_SOURCES.include?(@source_name)
+      raise "Invalid source name: #{@source_name}\nAvailable sources: #{AVAILABLE_SOURCES.map(&:to_s).join(", ")}"
+    end
+  end
+end

app/domain/sac_imports/membership_years_report.rb

@@ -8,14 +8,17 @@
 module SacImports
   class MembershipYearsReport
     REPORT_HEADERS = [
-      :navision_membership_number, :navision_name,
-      :navision_membership_years, :hitobito_membership_years,
-      :diff, :errors
+      :navision_membership_number,
+      :navision_name,
+      :navision_membership_years,
+      :hitobito_membership_years,
+      :diff,
+      :errors
     ].freeze
 
     def initialize(output: $stdout)
       @output = output
-      @source_file = CsvSourceFile.new(:NAV1)
+      @source_file = CsvSource.new(:NAV1)
       @csv_report = CsvReport.new(:"6_membership_years_report", REPORT_HEADERS)
     end
 
@@ -33,14 +36,14 @@ def create
     def process_row(row)
       @output.print "Reading row #{row[:navision_name]} ..."
       person = @hitobito_people[row[:navision_id].to_i]
-      @csv_report.add_row(
-        {navision_membership_number: row[:navision_id],
-         navision_name: row[:navision_name],
-         navision_membership_years: row[:navision_membership_years],
-         hitobito_membership_years: person&.membership_years,
-         diff: membership_years_diff(row[:navision_membership_years], person&.membership_years),
-         errors: errors_for(person)}
-      )
+      @csv_report.add_row({
+        navision_membership_number: row[:navision_id],
+        navision_name: row[:navision_name],
+        navision_membership_years: row[:navision_membership_years],
+        hitobito_membership_years: person&.membership_years,
+        diff: membership_years_diff(row[:navision_membership_years], person&.membership_years),
+        errors: errors_for(person)
+      })
       @output.print " processed.\n"
     end
 

app/domain/sac_imports/people_importer.rb

@@ -5,162 +5,30 @@
 #  or later. See the COPYING file at the top-level directory or at
 #  https://github.com/hitobito/hitobito_sac_cas.
 
-require Rails.root.join("lib", "import", "xlsx_reader.rb")
-
 module SacImports
   class PeopleImporter
-    class_attribute :headers, default: {
-      navision_id: "Mitgliedernummer",
-      person_type: "Personentyp",
-      salutation: "Anredecode",
-      name: "Name",
-      first_name: "Vorname",
-      last_name: "Nachname",
-      address_supplement: "Adresszusatz",
-      address: "Adresse",
-      country: "Länder-/Regionscode",
-      town: "Ort",
-      zip_code: "PLZ",
-      email: "E-Mail",
-      postfach: "Postfach",
-      phone: "Telefon",
-      phone_direct: "Telefon direkt",
-      phone_mobile: "Mobiltelefon",
-      birthday: "Geburtsdatum",
-      gender: "Geschlecht",
-      language: "Sprachcode"
-    }.freeze
-
-    class_attribute :sheet_name, default: "conv"
-
-    attr_reader :path, :skip_existing, :output, :errors, :invalid_emails, :processed_rows_count
+    REPORT_HEADERS = [
+      :navision_membership_number,
+      :navision_name,
+      :errors
+    ]
 
-    def initialize(path, skip_existing: false, output: $stdout)
-      @path = path
-      @skip_existing = skip_existing
+    def initialize(output: $stdout)
       @output = output
-      @errors = []
-      @invalid_emails = []
-      @processed_rows_count = 0
-    end
-
-    def import!
-      with_file do
-        each_row do |row|
-          next if skip?(row)
-
-          import_row(row)
-        end
-      end
-    ensure
-      print_errors
-      print_invalid_emails
-      print_summary
-    end
-
-    def each_row
-      without_query_logging do
-        Import::XlsxReader.read(path, sheet_name, headers: headers) do |row|
-          yield row unless row.compact.empty?
-          @processed_rows_count += 1
-        end
-      end
-    end
-
-    private
-
-    def with_file
-      return yield if path.exist?
-
-      output.puts "\nFAILED: Cannot read #{path.to_path}"
-    end
-
-    def navision_id(row)
-      row[:navision_id]
-    end
-
-    def existing_emails
-      @existing_emails ||= ::Person.pluck("DISTINCT(email)").compact.sort
-    end
-
-    def existing_people_ids
-      @existing_people_ids ||= Set.new(::Person.pluck(:id))
-    end
-
-    def skip?(row)
-      skip_existing && existing_people_ids.include?(navision_id(row).to_i)
+      @source_file = CsvSource.new(:NAV1)
+      @csv_report = CsvReport.new(:"1_people", REPORT_HEADERS)
     end
 
-    def import_row(row)
-      entry = SacImports::PersonEntry.new(row, group: contact_role_group, emails: existing_emails)
-
-      if entry.valid?
-        import_person(entry)
-      elsif only_invalid_email?(entry)
-        import_person_without_email(entry)
-      else
-        errors << entry.errors
+    def create
+      data = @source_file.rows
+      data.each do |row|
+        process_row(row)
       end
+      @csv_report.finalize(output: @output)
     end
 
-    def import_person(entry)
-      entry.import!
-      existing_emails << entry.email if entry.email
-      output.puts "Finished importing #{entry}"
-    rescue ActiveRecord::RecordInvalid => e
-      errors << "CAN NOT IMPORT ROW WITH NAVISION ID: #{navision_id(row).inspect}\n#{e.message}"
-    end
-
-    def import_person_without_email(entry)
-      @invalid_emails << "#{entry}: #{entry.email}"
-      entry.person.email = nil
-      import_person(entry)
-    end
-
-    def only_invalid_email?(entry)
-      entry.person.errors.attribute_names == [:email]
-    end
-
-    def contact_role_group
-      @contact_role_group ||= Group::ExterneKontakte.find_or_create_by!(
-        name: "Navision Import",
-        parent_id: Group::SacCas.first!.id
-      )
-    end
-
-    def without_query_logging
-      old_logger = ActiveRecord::Base.logger
-      ActiveRecord::Base.logger = nil
-      yield
-      ActiveRecord::Base.logger = old_logger
-    end
-
-    def print_summary
-      output.puts "\nProcessed #{@processed_rows_count} rows"
-      count = contact_role_group.roles.count
-      output.puts "#{contact_role_group} hat #{count} Rollen"
-    end
-
-    def print_errors
-      output_list("Die folgenden #{errors.size} Personen waren ungültig:", errors)
-    end
-
-    def print_invalid_emails
-      output_list("Die folgenden #{invalid_emails.size} Emails waren ungültig:", invalid_emails)
-    end
-
-    def output_list(text, list)
-      return if list.empty?
-
-      output.puts text
-      list.each { |item| output.puts " #{item}" }
-    end
-
-    def without_reset_primary_group
-      Role.skip_callback(:destroy, :after, :reset_primary_group, raise: false)
-      yield
-    ensure
-      Role.set_callback(:destroy, :after, :reset_primary_group)
+    def process_row(row)
+      @output.print("Reading row #{row[:navision_name]} ...")
     end
   end
 end