From 2ecf4c7c2f584139bd807459812ec350ec5c7c23 Mon Sep 17 00:00:00 2001 From: moveson Date: Fri, 20 Oct 2023 07:37:26 -0600 Subject: [PATCH] Additional changes to ease ultrasignup imports --- app/models/proto_record.rb | 1 + app/parameters/effort_parameters.rb | 37 ++++++++++--------- lib/etl/transformable.rb | 8 ++++ spec/fixtures/files/ultrasignup_efforts.csv | 2 +- .../etl/extractors/csv_file_strategy_spec.rb | 4 -- .../event_group_entrants_strategy_spec.rb | 6 ++- 6 files changed, 34 insertions(+), 24 deletions(-) diff --git a/app/models/proto_record.rb b/app/models/proto_record.rb index 4c5918477..75c1ecf6f 100644 --- a/app/models/proto_record.rb +++ b/app/models/proto_record.rb @@ -59,6 +59,7 @@ def run_specific_transforms(model, options) case model when :effort event = options[:event] + transfer_identified_gender! normalize_gender! normalize_country_code! normalize_state_code! diff --git a/app/parameters/effort_parameters.rb b/app/parameters/effort_parameters.rb index b8ee0f4cc..2fd3dd3f4 100644 --- a/app/parameters/effort_parameters.rb +++ b/app/parameters/effort_parameters.rb @@ -43,26 +43,29 @@ def self.permitted [:id, :slug, :event_id, :person_id, :participant_id, :first_name, :last_name, :gender, :wave, :bib_number, :age, :birthdate, :city, :state_code, :country_code, :finished, :start_time, :scheduled_start_time, :scheduled_start_time_local, :scheduled_start_offset, :beacon_url, :report_url, :photo, :phone, :email, :checked_in, :emergency_contact, :emergency_phone, - :comments, {split_times_attributes: [*SplitTimeParameters.permitted]}] + :comments, { split_times_attributes: [*SplitTimeParameters.permitted] }] end def self.mapping - {first: :first_name, - firstname: :first_name, - last: :last_name, - lastname: :last_name, - name: :full_name, - state: :state_code, - country: :country_code, - sex: :gender, - bib: :bib_number, - "bib_#": :bib_number, - dob: :birthdate, - birthday: :birthdate, - birth_date: :birthdate, - date_of_birth: :birthdate, - emergency_name: :emergency_contact, - start_time: :scheduled_start_time_local} + { + first: :first_name, + firstname: :first_name, + last: :last_name, + lastname: :last_name, + name: :full_name, + state: :state_code, + country: :country_code, + sex: :gender, + bib: :bib_number, + "bib_#": :bib_number, + dob: :birthdate, + birthday: :birthdate, + birth_date: :birthdate, + date_of_birth: :birthdate, + emergency_name: :emergency_contact, + start_time: :scheduled_start_time_local, + announcer_notes: :comments, + } end def self.unique_key diff --git a/lib/etl/transformable.rb b/lib/etl/transformable.rb index 8790c3811..9b9e28257 100644 --- a/lib/etl/transformable.rb +++ b/lib/etl/transformable.rb @@ -144,6 +144,8 @@ def normalize_gender! "male" when "f" "female" + when "x" + "nonbinary" when "n" "nonbinary" end @@ -212,6 +214,12 @@ def strip_white_space! end end + def transfer_identified_gender! + return unless self[:identified_gender].present? + + self[:gender] = delete_field(:identified_gender) + end + def underscore_keys! to_h.keys.each do |key| underscored_key = key.to_s.underscore.to_sym diff --git a/spec/fixtures/files/ultrasignup_efforts.csv b/spec/fixtures/files/ultrasignup_efforts.csv index 040f7f530..a811fa142 100644 --- a/spec/fixtures/files/ultrasignup_efforts.csv +++ b/spec/fixtures/files/ultrasignup_efforts.csv @@ -1,3 +1,3 @@ Order ID,Registration Date,distance,Quantity,Price,Price Option,order_type,Coupon,First Name,Last Name,gender,Identified Gender,Age,AgeGroup,DOB,Email,Address,City,State,Zip,Country,Phone,Removed,Bib,Captain,team_name,emergency_name,emergency_phone,statement_id,item_discount,order_tax,ultrasignup_fee,order_total,BBQ Ticket Meat 9,BBQ Ticket Veggie 9,Race logo 20,Mens Large 15,Mens Large 20,Mens Medium 15,Mens Medium 20,Mens Small 15,Mens Small 20,Mens XLarge 15,Mens XLarge 20,Mens XXL 15,One Size 25,Sticker 3,Unisex 2XL 35,Unisex Large 35,Unisex Medium 35,Unisex Small 35,Unisex XL 35,Womens Large 15,Womens Large 20,Womens Medium 15,Womens Medium 20,Womens Small 15,Womens Small 20,Womens XLarge 15,Womens XLarge 20,Announcer Notes,BBQ Preference,Is this your first attempt at this distance,Shirt Size,What name would you like printed on your bib 1234567,12/15/2022 7:00:10 AM,Half Marathon,1,85.0000,Half Marathon - $85,Credit Card,0.0000,Joe,Jackson,M,,40,,12/05/1982,jackson@gmail.com,222 Main Street,Durango,CO,80020,USA,3035551234,False,12,No,,Jane Jackson,3035552345,89410,0.0000,0.23,6.57,94.800000,,,,,,,,,,,,,,1,,,,,,,,,,,,,,,Meat,No,Men's Medium,Joe -1234568,12/15/2022 9:00:20 AM,Marathon,1,95.0000,Marathon - Registration $95,Credit Card,0.0000,Antonio,Banderas,M,,35,,08/15/1988,antonio@gmail.com,"43 Main Avenue, Unit 4",San Quentin,CO,80003,USA,3035556788,False,13,No,,Jimmy Banderas,3035554456,89410,0.0000,0.00,6.96,101.960000,,,,,,,,,,,,,,,,,,,,,,,,,,,,Greek,Meat,No,Men's Medium,Antonio +1234568,12/15/2022 9:00:20 AM,Marathon,1,95.0000,Marathon - Registration $95,Credit Card,0.0000,Antonio,Banderas,M,X,35,,08/15/1988,antonio@gmail.com,"43 Main Avenue, Unit 4",San Quentin,CO,80003,USA,3035556788,False,13,No,,Jimmy Banderas,3035554456,89410,0.0000,0.00,6.96,101.960000,,,,,,,,,,,,,,,,,,,,,,,,,,,,Greek,Meat,No,Men's Medium,Antonio diff --git a/spec/lib/etl/extractors/csv_file_strategy_spec.rb b/spec/lib/etl/extractors/csv_file_strategy_spec.rb index ab478fe4e..98a55a4b9 100644 --- a/spec/lib/etl/extractors/csv_file_strategy_spec.rb +++ b/spec/lib/etl/extractors/csv_file_strategy_spec.rb @@ -49,10 +49,6 @@ record = raw_data.first.to_h expect(record.keys).to match_array(expected_array) - - raw_data.each do |record| - pp record.to_h - end end end diff --git a/spec/lib/etl/transformers/event_group_entrants_strategy_spec.rb b/spec/lib/etl/transformers/event_group_entrants_strategy_spec.rb index 2c7ffb24f..3a3855c9b 100644 --- a/spec/lib/etl/transformers/event_group_entrants_strategy_spec.rb +++ b/spec/lib/etl/transformers/event_group_entrants_strategy_spec.rb @@ -117,7 +117,7 @@ let(:parsed_structs) do [ OpenStruct.new(:Order_ID=>1234567, :Registration_Date=>"12/15/2022 7:00:10 AM", :distance=>struct_1_event_name, :Quantity=>1, :Price=>85.0, :Price_Option=>"Half Marathon - $85", :order_type=>"Credit Card", :Coupon=>0.0, :First_Name=>"Joe", :Last_Name=>"Jackson", :gender=>"M", :Identified_Gender=>"", :Age=>40, :AgeGroup=>"", :DOB=>"12/05/1982", :Email=>"jackson@gmail.com", :Address=>"222 Main Street", :City=>"Durango", :State=>"CO", :Zip=>80020, :Country=>"USA", :Phone=>3035551234, :Removed=>"False", :Bib=>12, :Captain=>"No", :team_name=>"", :emergency_name=>"Jane Jackson", :emergency_phone=>3035552345, :statement_id=>89410, :item_discount=>0.0, :order_tax=>0.23, :ultrasignup_fee=>6.57, :order_total=>94.8, :BBQ_Ticket_Meat_9=>"", :BBQ_Ticket_Veggie_9=>"", :Race_logo_20=>"", :Mens_Large_15=>"", :Mens_Large_20=>"", :Mens_Medium_15=>"", :Mens_Medium_20=>"", :Mens_Small_15=>"", :Mens_Small_20=>"", :Mens_XLarge_15=>"", :Mens_XLarge_20=>"", :Mens_XXL_15=>"", :One_Size_25=>"", :Sticker_3=>1, :Unisex_2XL_35=>"", :Unisex_Large_35=>"", :Unisex_Medium_35=>"", :Unisex_Small_35=>"", :Unisex_XL_35=>"", :Womens_Large_15=>"", :Womens_Large_20=>"", :Womens_Medium_15=>"", :Womens_Medium_20=>"", :Womens_Small_15=>"", :Womens_Small_20=>"", :Womens_XLarge_15=>"", :Womens_XLarge_20=>"", :Announcer_Notes=>"", :BBQ_Preference=>"Meat", :Is_this_your_first_attempt_at_this_distance=>"No", :Shirt_Size=>"Men's Medium", :What_name_would_you_like_printed_on_your_bib=>"Joe"), - OpenStruct.new(:Order_ID=>1234568, :Registration_Date=>"12/15/2022 9:00:20 AM", :distance=>struct_2_event_name, :Quantity=>1, :Price=>95.0, :Price_Option=>"Marathon - Registration $95", :order_type=>"Credit Card", :Coupon=>0.0, :First_Name=>"Antonio", :Last_Name=>"Banderas", :gender=>"M", :Identified_Gender=>"", :Age=>35, :AgeGroup=>"", :DOB=>"08/15/1988", :Email=>"antonio@gmail.com", :Address=>"43 Main Avenue, Unit 4", :City=>"San Quentin", :State=>"CO", :Zip=>80003, :Country=>"USA", :Phone=>3035556788, :Removed=>"False", :Bib=>13, :Captain=>"No", :team_name=>"", :emergency_name=>"Jimmy Banderas", :emergency_phone=>3035554456, :statement_id=>89410, :item_discount=>0.0, :order_tax=>0.0, :ultrasignup_fee=>6.96, :order_total=>101.96, :BBQ_Ticket_Meat_9=>"", :BBQ_Ticket_Veggie_9=>"", :Race_logo_20=>"", :Mens_Large_15=>"", :Mens_Large_20=>"", :Mens_Medium_15=>"", :Mens_Medium_20=>"", :Mens_Small_15=>"", :Mens_Small_20=>"", :Mens_XLarge_15=>"", :Mens_XLarge_20=>"", :Mens_XXL_15=>"", :One_Size_25=>"", :Sticker_3=>"", :Unisex_2XL_35=>"", :Unisex_Large_35=>"", :Unisex_Medium_35=>"", :Unisex_Small_35=>"", :Unisex_XL_35=>"", :Womens_Large_15=>"", :Womens_Large_20=>"", :Womens_Medium_15=>"", :Womens_Medium_20=>"", :Womens_Small_15=>"", :Womens_Small_20=>"", :Womens_XLarge_15=>"", :Womens_XLarge_20=>"", :Announcer_Notes=>"Greek", :BBQ_Preference=>"Meat", :Is_this_your_first_attempt_at_this_distance=>"No", :Shirt_Size=>"Men's Medium", :What_name_would_you_like_printed_on_your_bib=>"Antonio") + OpenStruct.new(:Order_ID=>1234568, :Registration_Date=>"12/15/2022 9:00:20 AM", :distance=>struct_2_event_name, :Quantity=>1, :Price=>95.0, :Price_Option=>"Marathon - Registration $95", :order_type=>"Credit Card", :Coupon=>0.0, :First_Name=>"Antonio", :Last_Name=>"Banderas", :gender=>"M", :Identified_Gender=>"X", :Age=>35, :AgeGroup=>"", :DOB=>"08/15/1988", :Email=>"antonio@gmail.com", :Address=>"43 Main Avenue, Unit 4", :City=>"San Quentin", :State=>"CO", :Zip=>80003, :Country=>"USA", :Phone=>3035556788, :Removed=>"False", :Bib=>13, :Captain=>"No", :team_name=>"", :emergency_name=>"Jimmy Banderas", :emergency_phone=>3035554456, :statement_id=>89410, :item_discount=>0.0, :order_tax=>0.0, :ultrasignup_fee=>6.96, :order_total=>101.96, :BBQ_Ticket_Meat_9=>"", :BBQ_Ticket_Veggie_9=>"", :Race_logo_20=>"", :Mens_Large_15=>"", :Mens_Large_20=>"", :Mens_Medium_15=>"", :Mens_Medium_20=>"", :Mens_Small_15=>"", :Mens_Small_20=>"", :Mens_XLarge_15=>"", :Mens_XLarge_20=>"", :Mens_XXL_15=>"", :One_Size_25=>"", :Sticker_3=>"", :Unisex_2XL_35=>"", :Unisex_Large_35=>"", :Unisex_Medium_35=>"", :Unisex_Small_35=>"", :Unisex_XL_35=>"", :Womens_Large_15=>"", :Womens_Large_20=>"", :Womens_Medium_15=>"", :Womens_Medium_20=>"", :Womens_Small_15=>"", :Womens_Small_20=>"", :Womens_XLarge_15=>"", :Womens_XLarge_20=>"", :Announcer_Notes=>"Greek", :BBQ_Preference=>"Meat", :Is_this_your_first_attempt_at_this_distance=>"No", :Shirt_Size=>"Men's Medium", :What_name_would_you_like_printed_on_your_bib=>"Antonio") ] end @@ -146,16 +146,18 @@ expect(first_proto_record[:bib_number]).to eq(12) expect(first_proto_record[:emergency_contact]).to eq("Jane Jackson") expect(first_proto_record[:emergency_phone]).to eq(3035552345) + expect(first_proto_record[:comments]).to eq("") expect(second_proto_record[:first_name]).to eq("Antonio") expect(second_proto_record[:last_name]).to eq("Banderas") - expect(second_proto_record[:gender]).to eq("male") + expect(second_proto_record[:gender]).to eq("nonbinary") # transferred from :identified_gender expect(second_proto_record[:birthdate]).to eq("1988-08-15") expect(second_proto_record[:state_code]).to eq("CO") expect(second_proto_record[:country_code]).to eq("US") expect(second_proto_record[:bib_number]).to eq(13) expect(second_proto_record[:emergency_contact]).to eq("Jimmy Banderas") expect(second_proto_record[:emergency_phone]).to eq(3035554456) + expect(second_proto_record[:comments]).to eq("Greek") end end