SacImports changes (#1266)

* refactoring
* implements more importers

---------

Co-authored-by: Andreas Maierhofer <[email protected]>

.editorconfig

@@ -22,4 +22,4 @@ trim_trailing_whitespace = false
 
 [*.rb]
 max_line_length = 100
-ij_wrap_on_typing = true
+ij_wrap_on_typing = false

app/abilities/sac_cas/ability.rb

@@ -19,12 +19,12 @@ def define_user_abilities(...)
   end
 
   def prevent_changes_to_newsletter_mailing_list
-    with_options(internal_key: SacCas::NEWSLETTER_MAILING_LIST_INTERNAL_KEY) do
+    with_options(internal_key: SacCas::MAILING_LIST_SAC_NEWSLETTER_INTERNAL_KEY) do
       cannot [:destroy], MailingList
       cannot [:update], MailingList, [:subscribable_for, :subscribable_mode, :filter_chain]
     end
     cannot [:update, :destroy], Subscription, mailing_list: {
-      internal_key: SacCas::NEWSLETTER_MAILING_LIST_INTERNAL_KEY
+      internal_key: SacCas::MAILING_LIST_SAC_NEWSLETTER_INTERNAL_KEY
     }
     cannot [:update], Group, [:sac_newsletter_mailing_list_id]
   end

app/domain/sac_cas.rb

@@ -89,7 +89,12 @@ module SacCas
   ###
 
   MV_EMAIL = "[email protected]"
-  NEWSLETTER_MAILING_LIST_INTERNAL_KEY = "sac_newsletter"
+  MAILING_LIST_SAC_NEWSLETTER_INTERNAL_KEY = "sac_newsletter"
+  MAILING_LIST_SAC_INSIDE_INTERNAL_KEY = "sac_inside"
+  MAILING_LIST_TOURENLEITER_INTERNAL_KEY = "tourenleiter"
+  MAILING_LIST_DIE_ALPEN_PAPER_INTERNAL_KEY = "die_alpen_paper"
+  MAILING_LIST_DIE_ALPEN_DIGITAL_INTERNAL_KEY = "die_alpen_digital"
+  MAILING_LIST_SPENDENAUFRUFE_INTERNAL_KEY = "spendenaufrufe"
 
   AboCost = Data.define(:amount, :country)
   ABO_COSTS = {

app/domain/sac_exports/cluster_context.rb

@@ -0,0 +1,35 @@
+module SacExports
+  class ClusterContext
+    Credentials = Data.define(:username, :password)
+
+    attr_reader :namespace, :host, :env
+
+    def initialize(env)
+      @env = env
+      @host = "postgres05.cloud.puzzle.ch"
+      @namespace = "hit-sac-cas-#{env}"
+    end
+
+    def with_database
+      original = ActiveRecord::Base.remove_connection
+      ActiveRecord::Base.establish_connection(dbconfig)
+      yield
+    ensure
+      ActiveRecord::Base.establish_connection(original)
+    end
+
+    def credentials
+      @credentials ||= Credentials.new(**JSON.parse(`oc get --namespace #{namespace} secret pg-database-credentials -o json | jq '.data'`))
+    end
+
+    def dbconfig
+      @dbconfig ||= ActiveRecord::Base.configurations[:development].merge(
+        host: host,
+        database: namespace,
+        username: Base64.decode64(credentials.username),
+        password: Base64.decode64(credentials.password),
+        schema_search_path: :database
+      )
+    end
+  end
+end

app/domain/sac_exports/seed_generator.rb

@@ -0,0 +1,49 @@
+module SacExports
+  class SeedGenerator
+    def self.generate_custom_contents
+      new(CustomContent, keys: [:key]).generate
+      new(CustomContent::Translation, keys: [:custom_content_id, :locale]).generate
+
+      action_text_scope = ActionText::RichText.where(record_type: CustomContent::Translation.sti_name)
+      new(ActionText::RichText, scope: action_text_scope, keys: [:record_id, :record_type]).generate
+
+      new(ServiceToken, keys: [:token]).generate
+      new(Oauth::Application, keys: [:uid]).generate
+    end
+
+    def initialize(model, scope: model.all, keys: [])
+      @scope = scope
+      @model = model
+      @keys = keys
+      @mode = :seed
+      @file = Rails.root.join("tmp/#{model.table_name}_generated.rb")
+    end
+
+    def generate
+      code = generate_code
+      file.write(code)
+      puts "Generating code written to #{file}" # rubocop:disable Rails/Output
+    end
+
+    def generate_code
+      text = ""
+      text << "#{model}.#{mode}(#{seed_keys}"
+      rows.each do |row|
+        text << ",\n#{row}"
+      end
+      text << ")\n"
+    end
+
+    private
+
+    attr_reader :scope, :model, :keys, :mode, :file
+
+    def seed_keys
+      keys.map { |col| ":#{col}" }.join(" ,")
+    end
+
+    def rows
+      scope.map { |model| model.attributes.transform_values(&:to_s) }
+    end
+  end
+end

app/domain/sac_imports/ascii_table.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+#  Copyright (c) 2024, Schweizer Alpen-Club. This file is part of
+#  hitobito_sac_cas and licensed under the Affero General Public License version 3
+#  or later. See the COPYING file at the top-level directory or at
+#  https://github.com/hitobito/hitobito_sac_cas
+
+class SacImports::AsciiTable
+  def initialize(data)
+    @data = data
+  end
+
+  def to_s
+    table = StringIO.new
+    # Calculate column widths based on the length of the string representation of each cell
+    column_widths = @data.transpose.map { |column|
+      column.max_by { |cell| cell.to_s.length }
+        .to_s.length + 1
+    }
+
+    # Format the table header
+    header = @data.first.map.with_index do |cell, i|
+      if i == 0
+        cell.to_s.ljust(column_widths[i])
+      else
+        cell.to_s.rjust(column_widths[i] - 1)
+      end
+    end.join("|")
+    table.puts(header)
+
+    # Separate the header and body with a line
+    table.puts(separator(header.length))
+
+    # Format the table body
+    @data[1..].each do |row|
+      if row == "-"
+        table << separator(header.length)
+        next
+      end
+
+      row_formatted = row.map.with_index do |cell, i|
+        if i == 0
+          cell.to_s.ljust(column_widths[i])
+        else
+          cell.to_s.rjust(column_widths[i] - 1)
+        end
+      end
+      table.puts(row_formatted.join("|"))
+    end
+    table.puts(separator(header.length))
+    table.string
+  end
+
+  private
+
+  def separator(length) = "-" * length
+end

app/domain/sac_imports/cleanup.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+#  Copyright (c) 2024, Schweizer Alpen-Club. This file is part of
+#  hitobito_sac_cas and licensed under the Affero General Public License version 3
+#  or later. See the COPYING file at the top-level directory or at
+#  https://github.com/hitobito/hitobito_sac_cas
+
+module SacImports
+  class Cleanup
+    TASKS = [
+      RemoveNavisionRoles,
+      ValidatePrimaryGroup
+    ]
+
+    def run
+      TASKS.each do |task|
+        task.new.run
+      end
+    end
+  end
+end

app/domain/sac_imports/cleanup/remove_navision_roles.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+#  Copyright (c) 2024, Schweizer Alpen-Club. This file is part of
+#  hitobito_sac_cas and licensed under the Affero General Public License version 3
+#  or later. See the COPYING file at the top-level directory or at
+#  https://github.com/hitobito/hitobito_sac_cas
+
+module SacImports
+  class Cleanup::RemoveNavisionRoles
+    GROUP_NAME = "Navision Import"
+
+    def run
+      obsolete_roles.delete_all.tap do |count|
+        puts "Deleted #{count} obsolete #{GROUP_NAME} roles" if count.positive? # rubocop:disable Rails/Output
+      end
+    end
+
+    private
+
+    def obsolete_roles = navision_roles.where(person_id: other_roles.pluck(:person_id))
+
+    def navision_roles = roles.where(conditions)
+
+    def other_roles = roles.where.not(conditions)
+
+    def conditions = {groups: {type: Group::ExterneKontakte.sti_name, name: GROUP_NAME}}
+
+    def roles = Role.with_inactive.joins(:group)
+  end
+end

app/domain/sac_imports/cleanup/validate_primary_group.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+#  Copyright (c) 2024, Schweizer Alpen-Club. This file is part of
+#  hitobito_sac_cas and licensed under the Affero General Public License version 3
+#  or later. See the COPYING file at the top-level directory or at
+#  https://github.com/hitobito/hitobito_sac_cas
+
+module SacImports
+  class Cleanup::ValidatePrimaryGroup
+    OUTER_JOIN = "LEFT OUTER JOIN roles ON roles.person_id = people.id AND roles.group_id = primary_group_id"
+
+    def run
+      nullify_primary_group if people_without_roles.exists?
+      reset_primary_group if people_without_primary_group.exists?
+    end
+
+    private
+
+    def nullify_primary_group
+      puts "Nullifying primary_group_id for #{people_without_roles.count} People without roles" # rubocop:disable Rails/Output
+      people_without_roles.update_all(primary_group_id: nil)
+    end
+
+    def reset_primary_group
+      puts "Resetting primary_group_id for #{people_without_primary_group.count} People" # rubocop:disable Rails/Output
+      people_without_primary_group.find_each do |person|
+        ::People::UpdateAfterRoleChange.new(person.reload).set_first_primary_group
+      end
+    end
+
+    def people_without_roles = people.where.missing(:roles)
+
+    def people_without_active_roles = people.merge(Role.active).joins(OUTER_JOIN).where(roles: {group_id: nil})
+
+    def people_without_primary_group = people.merge(Role.active).joins(OUTER_JOIN).where(roles: {group_id: nil})
+
+    def people = Person.where.not(id: Person.root.id).where.not(primary_group_id: nil)
+  end
+end

app/domain/sac_imports/csv_report.rb

@@ -7,28 +7,44 @@
 
 class SacImports::CsvReport
   COLUMN_SEPARATOR = ";"
+  LOG_DIR = Rails.root.join("log", "sac_imports")
 
-  def initialize(sac_import_name, headers)
+  delegate :puts, to: :@output
+
+  def initialize(sac_import_name, headers, output: $stdout)
     @start_time = Time.zone.now
     @timestamp = format_time(Time.zone.now)
     @sac_import_name = sac_import_name
     @headers = headers
+    @output = output
     csv_init
+    log("Started: #{@timestamp}")
+    log("Database: #{ActiveRecord::Base.connection.current_database}")
+    output.puts "Report for #{@sac_import_name} started at #{@timestamp}."
+    output.puts "Report location: #{csv_file_path}"
+    output.puts "Log location: #{log_file_path}"
+    output.puts "Database: #{ActiveRecord::Base.connection.current_database}"
   end
 
   def add_row(row)
     csv_append(row)
   end
 
-  def finalize(output: $stdout)
+  def log(line)
+    @output.puts(line)
+    File.write(log_file_path, "#{line}\n", mode: "a")
+  end
+
+  def finalize
     log(
       "Started: #{@timestamp}, " \
         "completed: #{format_time(Time.zone.now)}, " \
-        "duration: #{format_duration} minutes"
+        "duration: #{format_duration}"
     )
-    output.puts "\n\n\nReport generated in #{format_duration}."
-    output.puts "Thank you for flying with SAC Imports."
-    output.puts "Report written to #{csv_file_path}"
+    @output.puts "\n\n\nReport generated in #{format_duration}."
+    @output.puts "Thank you for flying with SAC Imports."
+    @output.puts "Report written to #{csv_file_path}"
+    @output.puts "Log written to #{log_file_path}"
   end
 
   private
@@ -42,9 +58,8 @@ def log_dir
   end
 
   def create_log_dir
-    log_dir = Rails.root.join("log", "sac_imports")
-    log_dir.mkpath
-    log_dir
+    LOG_DIR.mkpath
+    LOG_DIR
   end
 
   def csv_init
@@ -61,15 +76,11 @@ def csv_append(row_content)
     end
   end
 
-  def log(line)
-    File.write(log_file_path, "#{line}\n", mode: "a")
-  end
-
   def csv_file_path
     @csv_file_path ||= "#{log_dir}/#{@sac_import_name}_#{@timestamp}.csv"
   end
 
   def log_file_path
-    @log_file_path ||= "#{log_dir}/#{@sac_import_name}.log"
+    @log_file_path ||= "#{log_dir}/#{@sac_import_name}_#{@timestamp}.log"
   end
 end