Merge pull request #3343 from DFE-Digital/better-csv

[CAPT-1957] Handle currency for eligble FE CSV upload

app/models/eligible_fe_providers_importer.rb

@@ -31,8 +31,8 @@ def delete_all_scope
   def row_to_hash(row)
     {
       ukprn: row.fetch("ukprn"),
-      max_award_amount: row.fetch("max_award_amount"),
-      lower_award_amount: row.fetch("lower_award_amount"),
+      max_award_amount: row.fetch("max_award_amount").gsub(/£|,|�/, ""),
+      lower_award_amount: row.fetch("lower_award_amount").gsub(/£|,|�/, ""),
       primary_key_contact_email_address: row.fetch("primary_key_contact_email_address"),
       academic_year:
     }

lib/csv_importer/base.rb

@@ -70,7 +70,8 @@ def parse_csv_file(file)
         errors.append("Select a file")
         nil
       else
-        CSV.read(file.to_io, headers: parse_headers, encoding: "BOM|UTF-8")
+        string = File.open(file.path, "r", encoding: "BOM|UTF-8").read.scrub
+        CSV.parse(string, headers: parse_headers)
       end
     rescue CSV::MalformedCSVError
       errors.append("The selected file must be a CSV")

spec/fixtures/files/eligible_fe_providers_illegal_encoding.csv

@@ -0,0 +1,11 @@
+ukprn,max_award_amount,lower_award_amount,primary_key_contact_email_address
+10083728,"�4,000","�2,000",[email protected]
+10000055,"�6,000","�3,000",[email protected]
+10004927,"�6,000","�3,000",[email protected]
+10057981,"�6,000","�3,000",[email protected]
+10000330,"�5,000","�2,500",[email protected]
+10082366,"�6,000","�3,000",[email protected]
+10000415,"�6,000","�3,000",[email protected]
+10000528,"�6,000","�3,000",[email protected]
+10000533,"�6,000","�3,000",[email protected]
+10000536,"�6,000","�3,000",[email protected]

spec/models/eligible_fe_providers_importer_spec.rb

@@ -114,5 +114,44 @@ def to_row(hash)
         end
       end
     end
+
+    context "when currency values has GBP symbols and thousand separators" do
+      before do
+        file.write correct_headers
+
+        3.times do
+          file.write to_row(attributes_for(:eligible_fe_provider).merge(max_award_amount: '"£6,000"', lower_award_amount: '"£3,000"'))
+        end
+
+        file.close
+      end
+
+      it "ignores superfluous characters and imports new records" do
+        expect { subject.run }.to change { EligibleFeProvider.count }.by(3)
+        expect(EligibleFeProvider.pluck(:max_award_amount).uniq).to eql([6000])
+        expect(EligibleFeProvider.pluck(:lower_award_amount).uniq).to eql([3000])
+      end
+
+      context "when there are existing records" do
+        before do
+          create(:eligible_fe_provider)
+          create(:eligible_fe_provider, academic_year: AcademicYear.next)
+        end
+
+        it "deletes them with new records" do
+          expect { subject.run }.to change { EligibleFeProvider.count }.by(2)
+        end
+      end
+    end
+
+    context "when file has illegal encoding" do
+      let(:file) { File.open(file_fixture("eligible_fe_providers_illegal_encoding.csv")) }
+
+      it "ignores superfluous characters and imports new records" do
+        expect { subject.run }.to change { EligibleFeProvider.count }.by(10)
+        expect(EligibleFeProvider.pluck(:max_award_amount).uniq.sort).to eql([4_000, 5_000, 6_000])
+        expect(EligibleFeProvider.pluck(:lower_award_amount).uniq.sort).to eql([2_000, 2_500, 3_000])
+      end
+    end
   end
 end