Additional changes to ease ultrasignup imports

app/models/proto_record.rb

@@ -59,6 +59,7 @@ def run_specific_transforms(model, options)
     case model
     when :effort
       event = options[:event]
+      transfer_identified_gender!
       normalize_gender!
       normalize_country_code!
       normalize_state_code!

app/parameters/effort_parameters.rb

@@ -43,26 +43,29 @@ def self.permitted
     [:id, :slug, :event_id, :person_id, :participant_id, :first_name, :last_name, :gender, :wave, :bib_number, :age, :birthdate,
      :city, :state_code, :country_code, :finished, :start_time, :scheduled_start_time, :scheduled_start_time_local,
      :scheduled_start_offset, :beacon_url, :report_url, :photo, :phone, :email, :checked_in, :emergency_contact, :emergency_phone,
-     :comments, {split_times_attributes: [*SplitTimeParameters.permitted]}]
+     :comments, { split_times_attributes: [*SplitTimeParameters.permitted] }]
   end
 
   def self.mapping
-    {first: :first_name,
-     firstname: :first_name,
-     last: :last_name,
-     lastname: :last_name,
-     name: :full_name,
-     state: :state_code,
-     country: :country_code,
-     sex: :gender,
-     bib: :bib_number,
-     "bib_#": :bib_number,
-     dob: :birthdate,
-     birthday: :birthdate,
-     birth_date: :birthdate,
-     date_of_birth: :birthdate,
-     emergency_name: :emergency_contact,
-     start_time: :scheduled_start_time_local}
+    {
+      first: :first_name,
+      firstname: :first_name,
+      last: :last_name,
+      lastname: :last_name,
+      name: :full_name,
+      state: :state_code,
+      country: :country_code,
+      sex: :gender,
+      bib: :bib_number,
+      "bib_#": :bib_number,
+      dob: :birthdate,
+      birthday: :birthdate,
+      birth_date: :birthdate,
+      date_of_birth: :birthdate,
+      emergency_name: :emergency_contact,
+      start_time: :scheduled_start_time_local,
+      announcer_notes: :comments,
+    }
   end
 
   def self.unique_key

lib/etl/transformable.rb

@@ -144,6 +144,8 @@ def normalize_gender!
                           "male"
                         when "f"
                           "female"
+                        when "x"
+                          "nonbinary"
                         when "n"
                           "nonbinary"
                         end
@@ -212,6 +214,12 @@ def strip_white_space!
       end
     end
 
+    def transfer_identified_gender!
+      return unless self[:identified_gender].present?
+
+      self[:gender] = delete_field(:identified_gender)
+    end
+
     def underscore_keys!
       to_h.keys.each do |key|
         underscored_key = key.to_s.underscore.to_sym

spec/fixtures/files/ultrasignup_efforts.csv

@@ -1,3 +1,3 @@
 Order ID,Registration Date,distance,Quantity,Price,Price Option,order_type,Coupon,First Name,Last Name,gender,Identified Gender,Age,AgeGroup,DOB,Email,Address,City,State,Zip,Country,Phone,Removed,Bib,Captain,team_name,emergency_name,emergency_phone,statement_id,item_discount,order_tax,ultrasignup_fee,order_total,BBQ Ticket  Meat  9,BBQ Ticket  Veggie  9,Race logo 20,Mens Large  15,Mens Large  20,Mens Medium  15,Mens Medium  20,Mens Small  15,Mens Small  20,Mens XLarge  15,Mens XLarge  20,Mens XXL  15,One Size   25,Sticker  3,Unisex 2XL  35,Unisex Large  35,Unisex Medium  35,Unisex Small  35,Unisex XL   35,Womens Large  15,Womens Large  20,Womens Medium  15,Womens Medium  20,Womens Small  15,Womens Small  20,Womens XLarge  15,Womens XLarge  20,Announcer Notes,BBQ Preference,Is this your first attempt at this distance,Shirt Size,What name would you like printed on your bib
 1234567,12/15/2022 7:00:10 AM,Half Marathon,1,85.0000,Half Marathon - $85,Credit Card,0.0000,Joe,Jackson,M,,40,,12/05/1982,[email protected],222 Main Street,Durango,CO,80020,USA,3035551234,False,12,No,,Jane Jackson,3035552345,89410,0.0000,0.23,6.57,94.800000,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,Meat,No,Men's Medium,Joe
-1234568,12/15/2022 9:00:20 AM,Marathon,1,95.0000,Marathon - Registration $95,Credit Card,0.0000,Antonio,Banderas,M,,35,,08/15/1988,[email protected],"43 Main Avenue, Unit 4",San Quentin,CO,80003,USA,3035556788,False,13,No,,Jimmy Banderas,3035554456,89410,0.0000,0.00,6.96,101.960000,,,,,,,,,,,,,,,,,,,,,,,,,,,,Greek,Meat,No,Men's Medium,Antonio
+1234568,12/15/2022 9:00:20 AM,Marathon,1,95.0000,Marathon - Registration $95,Credit Card,0.0000,Antonio,Banderas,M,X,35,,08/15/1988,[email protected],"43 Main Avenue, Unit 4",San Quentin,CO,80003,USA,3035556788,False,13,No,,Jimmy Banderas,3035554456,89410,0.0000,0.00,6.96,101.960000,,,,,,,,,,,,,,,,,,,,,,,,,,,,Greek,Meat,No,Men's Medium,Antonio

spec/lib/etl/extractors/csv_file_strategy_spec.rb

@@ -49,10 +49,6 @@
 
         record = raw_data.first.to_h
         expect(record.keys).to match_array(expected_array)
-
-        raw_data.each do |record|
-          pp record.to_h
-        end
       end
     end
 

spec/lib/etl/transformers/event_group_entrants_strategy_spec.rb

@@ -117,7 +117,7 @@
         let(:parsed_structs) do
           [
             OpenStruct.new(:Order_ID=>1234567, :Registration_Date=>"12/15/2022 7:00:10 AM", :distance=>struct_1_event_name, :Quantity=>1, :Price=>85.0, :Price_Option=>"Half Marathon - $85", :order_type=>"Credit Card", :Coupon=>0.0, :First_Name=>"Joe", :Last_Name=>"Jackson", :gender=>"M", :Identified_Gender=>"", :Age=>40, :AgeGroup=>"", :DOB=>"12/05/1982", :Email=>"[email protected]", :Address=>"222 Main Street", :City=>"Durango", :State=>"CO", :Zip=>80020, :Country=>"USA", :Phone=>3035551234, :Removed=>"False", :Bib=>12, :Captain=>"No", :team_name=>"", :emergency_name=>"Jane Jackson", :emergency_phone=>3035552345, :statement_id=>89410, :item_discount=>0.0, :order_tax=>0.23, :ultrasignup_fee=>6.57, :order_total=>94.8, :BBQ_Ticket_Meat_9=>"", :BBQ_Ticket_Veggie_9=>"", :Race_logo_20=>"", :Mens_Large_15=>"", :Mens_Large_20=>"", :Mens_Medium_15=>"", :Mens_Medium_20=>"", :Mens_Small_15=>"", :Mens_Small_20=>"", :Mens_XLarge_15=>"", :Mens_XLarge_20=>"", :Mens_XXL_15=>"", :One_Size_25=>"", :Sticker_3=>1, :Unisex_2XL_35=>"", :Unisex_Large_35=>"", :Unisex_Medium_35=>"", :Unisex_Small_35=>"", :Unisex_XL_35=>"", :Womens_Large_15=>"", :Womens_Large_20=>"", :Womens_Medium_15=>"", :Womens_Medium_20=>"", :Womens_Small_15=>"", :Womens_Small_20=>"", :Womens_XLarge_15=>"", :Womens_XLarge_20=>"", :Announcer_Notes=>"", :BBQ_Preference=>"Meat", :Is_this_your_first_attempt_at_this_distance=>"No", :Shirt_Size=>"Men's Medium", :What_name_would_you_like_printed_on_your_bib=>"Joe"),
-            OpenStruct.new(:Order_ID=>1234568, :Registration_Date=>"12/15/2022 9:00:20 AM", :distance=>struct_2_event_name, :Quantity=>1, :Price=>95.0, :Price_Option=>"Marathon - Registration $95", :order_type=>"Credit Card", :Coupon=>0.0, :First_Name=>"Antonio", :Last_Name=>"Banderas", :gender=>"M", :Identified_Gender=>"", :Age=>35, :AgeGroup=>"", :DOB=>"08/15/1988", :Email=>"[email protected]", :Address=>"43 Main Avenue, Unit 4", :City=>"San Quentin", :State=>"CO", :Zip=>80003, :Country=>"USA", :Phone=>3035556788, :Removed=>"False", :Bib=>13, :Captain=>"No", :team_name=>"", :emergency_name=>"Jimmy Banderas", :emergency_phone=>3035554456, :statement_id=>89410, :item_discount=>0.0, :order_tax=>0.0, :ultrasignup_fee=>6.96, :order_total=>101.96, :BBQ_Ticket_Meat_9=>"", :BBQ_Ticket_Veggie_9=>"", :Race_logo_20=>"", :Mens_Large_15=>"", :Mens_Large_20=>"", :Mens_Medium_15=>"", :Mens_Medium_20=>"", :Mens_Small_15=>"", :Mens_Small_20=>"", :Mens_XLarge_15=>"", :Mens_XLarge_20=>"", :Mens_XXL_15=>"", :One_Size_25=>"", :Sticker_3=>"", :Unisex_2XL_35=>"", :Unisex_Large_35=>"", :Unisex_Medium_35=>"", :Unisex_Small_35=>"", :Unisex_XL_35=>"", :Womens_Large_15=>"", :Womens_Large_20=>"", :Womens_Medium_15=>"", :Womens_Medium_20=>"", :Womens_Small_15=>"", :Womens_Small_20=>"", :Womens_XLarge_15=>"", :Womens_XLarge_20=>"", :Announcer_Notes=>"Greek", :BBQ_Preference=>"Meat", :Is_this_your_first_attempt_at_this_distance=>"No", :Shirt_Size=>"Men's Medium", :What_name_would_you_like_printed_on_your_bib=>"Antonio")
+            OpenStruct.new(:Order_ID=>1234568, :Registration_Date=>"12/15/2022 9:00:20 AM", :distance=>struct_2_event_name, :Quantity=>1, :Price=>95.0, :Price_Option=>"Marathon - Registration $95", :order_type=>"Credit Card", :Coupon=>0.0, :First_Name=>"Antonio", :Last_Name=>"Banderas", :gender=>"M", :Identified_Gender=>"X", :Age=>35, :AgeGroup=>"", :DOB=>"08/15/1988", :Email=>"[email protected]", :Address=>"43 Main Avenue, Unit 4", :City=>"San Quentin", :State=>"CO", :Zip=>80003, :Country=>"USA", :Phone=>3035556788, :Removed=>"False", :Bib=>13, :Captain=>"No", :team_name=>"", :emergency_name=>"Jimmy Banderas", :emergency_phone=>3035554456, :statement_id=>89410, :item_discount=>0.0, :order_tax=>0.0, :ultrasignup_fee=>6.96, :order_total=>101.96, :BBQ_Ticket_Meat_9=>"", :BBQ_Ticket_Veggie_9=>"", :Race_logo_20=>"", :Mens_Large_15=>"", :Mens_Large_20=>"", :Mens_Medium_15=>"", :Mens_Medium_20=>"", :Mens_Small_15=>"", :Mens_Small_20=>"", :Mens_XLarge_15=>"", :Mens_XLarge_20=>"", :Mens_XXL_15=>"", :One_Size_25=>"", :Sticker_3=>"", :Unisex_2XL_35=>"", :Unisex_Large_35=>"", :Unisex_Medium_35=>"", :Unisex_Small_35=>"", :Unisex_XL_35=>"", :Womens_Large_15=>"", :Womens_Large_20=>"", :Womens_Medium_15=>"", :Womens_Medium_20=>"", :Womens_Small_15=>"", :Womens_Small_20=>"", :Womens_XLarge_15=>"", :Womens_XLarge_20=>"", :Announcer_Notes=>"Greek", :BBQ_Preference=>"Meat", :Is_this_your_first_attempt_at_this_distance=>"No", :Shirt_Size=>"Men's Medium", :What_name_would_you_like_printed_on_your_bib=>"Antonio")
           ]
         end
 
@@ -146,16 +146,18 @@
           expect(first_proto_record[:bib_number]).to eq(12)
           expect(first_proto_record[:emergency_contact]).to eq("Jane Jackson")
           expect(first_proto_record[:emergency_phone]).to eq(3035552345)
+          expect(first_proto_record[:comments]).to eq("")
 
           expect(second_proto_record[:first_name]).to eq("Antonio")
           expect(second_proto_record[:last_name]).to eq("Banderas")
-          expect(second_proto_record[:gender]).to eq("male")
+          expect(second_proto_record[:gender]).to eq("nonbinary") # transferred from :identified_gender
           expect(second_proto_record[:birthdate]).to eq("1988-08-15")
           expect(second_proto_record[:state_code]).to eq("CO")
           expect(second_proto_record[:country_code]).to eq("US")
           expect(second_proto_record[:bib_number]).to eq(13)
           expect(second_proto_record[:emergency_contact]).to eq("Jimmy Banderas")
           expect(second_proto_record[:emergency_phone]).to eq(3035554456)
+          expect(second_proto_record[:comments]).to eq("Greek")
         end
       end