Skip to content

Commit

Permalink
CsvImporter.import() accepts a path instead of a string
Browse files Browse the repository at this point in the history
  • Loading branch information
Alex Dolski committed Dec 13, 2023
1 parent 754e447 commit 062fc4c
Show file tree
Hide file tree
Showing 3 changed files with 184 additions and 52 deletions.
32 changes: 21 additions & 11 deletions app/util/csv_importer.rb
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ class CsvImporter
##
# Imports items from a CSV string.
#
# @param csv [String] CSV string.
# @param pathname [String] Pathname of the CSV file.
# @param file_paths [Enumerable<String>] Absolute paths of files that are
# referenced in the CSV's `files`
# column.
Expand All @@ -137,37 +137,46 @@ class CsvImporter
# @param task [Task] Supply to receive status updates.
# @return [void]
#
def import(csv:,
def import(pathname:,
file_paths: [],
submitter:,
primary_collection:,
imported_items: [],
print_progress: false,
task: nil)
rows = CSV.parse(csv)
num_rows = rows.length - 1 # exclude header
num_rows = 0
File.foreach(pathname) do
num_rows += 1
end
progress = print_progress ? Progress.new(num_rows) : nil
# Work inside a transaction to avoid any incompletely created items. If
# any items fail, we want the whole import to fail.
Import.transaction do
begin
validate_header(row: rows[0],
submission_profile: primary_collection.effective_submission_profile)
rows[1..].each_with_index do |row, row_index|
row_index = 0
header_row = nil
CSV.foreach(pathname) do |row|
if row_index == 0
header_row = row
validate_header(row: header_row,
submission_profile: primary_collection.effective_submission_profile)
row_index += 1
next
end
# Create or update the item.
item_id = row[REQUIRED_COLUMNS.index("id")]&.strip
raise ArgumentError, "Missing item ID" if item_id.blank?
if item_id == NEW_ITEM_INDICATOR
item = create_item(submitter: submitter,
primary_collection: primary_collection,
element_names: rows[0][REQUIRED_COLUMNS.length..],
element_names: header_row[REQUIRED_COLUMNS.length..],
row: row,
file_paths: file_paths)
else
item = Item.find(item_id)
item = update_item(item: item,
submitter: submitter,
element_names: rows[0][REQUIRED_COLUMNS.length..],
element_names: header_row[REQUIRED_COLUMNS.length..],
row: row,
file_paths: file_paths)
end
Expand All @@ -176,8 +185,9 @@ def import(csv:,
handle: item.handle&.handle
}
status_text = "Importing #{num_rows} items from CSV"
progress&.report(row_index + 1, status_text)
task&.progress(row_index + 1 / (num_rows - 1).to_f,
row_index += 1
progress&.report(row_index, status_text)
task&.progress(row_index / (num_rows - 1).to_f,
status_text: status_text)
end
rescue => e
Expand Down
4 changes: 2 additions & 2 deletions app/util/importer.rb
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ def import(import, submitter)
return Import::Format::SAF
elsif root_files.find{ |f| f.downcase.end_with?(".csv") }
import.task.update!(status_text: "Importing items from CSV package")
CsvImporter.new.import(csv: File.read(root_files.find{ |f| f.downcase.end_with?(".csv") }),
CsvImporter.new.import(pathname: root_files.find{ |f| f.downcase.end_with?(".csv") },
file_paths: Dir[tmpdir + "/**/*"],
submitter: submitter,
primary_collection: import.collection,
Expand All @@ -84,7 +84,7 @@ def import(import, submitter)
end
elsif file.split(".").last.downcase == "csv"
import.task.update!(status_text: "Importing items from CSV file")
CsvImporter.new.import(csv: File.read(file),
CsvImporter.new.import(pathname: file,
submitter: submitter,
primary_collection: import.collection,
imported_items: [],
Expand Down
Loading

0 comments on commit 062fc4c

Please sign in to comment.