Improve CSV parsing, especially when encoding is not UTF-8

lib/modsvaskr/encoding.rb

@@ -0,0 +1,29 @@
+module Modsvaskr
+
+  module Encoding
+
+    # Convert a string to UTF-8
+    #
+    # Parameters::
+    # * *str* (String): The string to convert
+    # Result::
+    # * String: The converted string
+    def self.to_utf8(str)
+      orig_encoding = str.encoding
+      encoding = nil
+      begin
+        encoding = %w[
+          UTF-8
+          Windows-1252
+          ISO-8859-1
+        ].find { |search_encoding| str.force_encoding(search_encoding).valid_encoding? }
+      ensure
+        str.force_encoding(orig_encoding)
+      end
+      raise "Unknown encoding for string #{str[0..127].inspect}" if encoding.nil?
+      str.encode('UTF-8', encoding)
+    end
+
+  end
+
+end

lib/modsvaskr/tests_suites/exterior_cell.rb

@@ -26,7 +26,7 @@ def discover_tests
         # Hash< String, Hash< String, Array<[Integer, Integer]> > >
         exterior_cells = {}
         @game.xedit.run_script('DumpInfo', only_once: true)
-        CSV.read("#{@game.xedit.install_path}/Edit Scripts/Modsvaskr_ExportedDumpInfo.csv", encoding: 'windows-1251:utf-8').each do |row|
+        @game.xedit.parse_csv('Modsvaskr_ExportedDumpInfo') do |row|
           esp_name, record_type = row[0..1]
           if record_type.downcase == 'cell'
             cell_type, cell_name, cell_x, cell_y = row[3..6]

lib/modsvaskr/tests_suites/interior_cell.rb

@@ -26,7 +26,7 @@ def discover_tests
         # Hash< String, Array<String> >
         interior_cells = {}
         @game.xedit.run_script('DumpInfo', only_once: true)
-        CSV.read("#{@game.xedit.install_path}/Edit Scripts/Modsvaskr_ExportedDumpInfo.csv", encoding: 'windows-1251:utf-8').each do |row|
+        @game.xedit.parse_csv('Modsvaskr_ExportedDumpInfo') do |row|
           esp_name, record_type = row[0..1]
           if record_type.downcase == 'cell'
             cell_type, cell_name = row[3..4]

lib/modsvaskr/tests_suites/npc.rb

@@ -24,7 +24,7 @@ def in_game_tests_suite
       def discover_tests
         tests = {}
         @game.xedit.run_script('DumpInfo', only_once: true)
-        CSV.read("#{@game.xedit.install_path}/Edit Scripts/Modsvaskr_ExportedDumpInfo.csv", encoding: 'windows-1251:utf-8').each do |row|
+        @game.xedit.parse_csv('Modsvaskr_ExportedDumpInfo') do |row|
           tests["#{row[0].downcase}/#{row[2].to_i(16)}"] = {
             name: "Take screenshot of #{row[0]} - #{row[3]}"
           } if row[1].downcase == 'npc_'

lib/modsvaskr/tests_suites/npc_head.rb

@@ -24,7 +24,7 @@ def in_game_tests_suite
       def discover_tests
         tests = {}
         @game.xedit.run_script('DumpInfo', only_once: true)
-        CSV.read("#{@game.xedit.install_path}/Edit Scripts/Modsvaskr_ExportedDumpInfo.csv", encoding: 'windows-1251:utf-8').each do |row|
+        @game.xedit.parse_csv('Modsvaskr_ExportedDumpInfo') do |row|
           tests["#{row[0].downcase}/#{row[2].to_i(16)}"] = {
             name: "Take head screenshot of #{row[0]} - #{row[3]}"
           } if row[1].downcase == 'npc_'

lib/modsvaskr/xedit.rb

@@ -1,3 +1,4 @@
+require 'modsvaskr/encoding'
 require 'modsvaskr/run_cmd'
 
 module Modsvaskr
@@ -47,6 +48,17 @@ def run_script(script, only_once: false)
       end
     end
 
+    # Parse a CSV that has been dumped by a previous run of xEdit
+    #
+    # Parameters::
+    # * *csv* (String): Name of the CSV file (from Edit Scripts), without .csv
+    # * *row_block* (Proc): Code called for each CSV row
+    #   Parameters::
+    #   * *row* (Array<String>): CSV row
+    def parse_csv(csv, &row_block)
+      CSV.parse(Encoding.to_utf8(File.read("#{install_path}/Edit Scripts/#{csv}.csv", mode: 'rb'))).each(&row_block)
+    end
+
   end
 
 end