diff --git a/grails-app/controllers/au/org/ala/ecodata/MetadataController.groovy b/grails-app/controllers/au/org/ala/ecodata/MetadataController.groovy
index f563c13bd..f0abade58 100644
--- a/grails-app/controllers/au/org/ala/ecodata/MetadataController.groovy
+++ b/grails-app/controllers/au/org/ala/ecodata/MetadataController.groovy
@@ -9,6 +9,7 @@ import org.springframework.web.multipart.MultipartFile
import static au.org.ala.ecodata.Status.DELETED
class MetadataController {
+ static responseFormats = ['json']
def metadataService, activityService, commonService, projectService, webService
@@ -235,17 +236,22 @@ class MetadataController {
if (file && outputName) {
def data = metadataService.excelWorkbookToMap(file.inputStream, outputName, true)
+ def status = 200
def result
- if (!data) {
- response.status = 400
+ if (data.error) {
+ status = 500
+ data.status = 500
+ result = data
+ }
+ else if (!data.success) {
+ status = 400
result = [status:400, error:'No data was found that matched the output description identified by the type parameter, please check the template you used to upload the data. ']
}
else {
- result = [status: 200, data:data]
+ result = [status: 200, data:data.success]
}
- render result as JSON
-
+ respond(result, status: status)
}
else {
response.status = 400
diff --git a/grails-app/i18n/messages.properties b/grails-app/i18n/messages.properties
index 6ed321f16..606c2b72a 100644
--- a/grails-app/i18n/messages.properties
+++ b/grails-app/i18n/messages.properties
@@ -309,4 +309,8 @@ projectAcitivity.attribution={0}. ({1}) {2} dataset download. Retrieved from {3}
activityForm.invalidIndex=Form template {0} has invalid index fields {1}
activityForm.latestVersionIsInDraft = Cannot create a new draft of a form when the current version is a draft
-user.userHubs.hubId.unique=Each UserHub must have a unique hubId
\ No newline at end of file
+user.userHubs.hubId.unique=Each UserHub must have a unique hubId
+
+bulkimport.conversionToObjectError=Error parsing data into an object in serial number {0}
+bulkimport.errorGroupBySerialNumber=Error making nested object from multiple rows in serial number {0}
+bulkimport.conversionToFormSectionError=Error parsing data for form section (output) - {0}
\ No newline at end of file
diff --git a/grails-app/services/au/org/ala/ecodata/MetadataService.groovy b/grails-app/services/au/org/ala/ecodata/MetadataService.groovy
index ff5ff114c..b82e6b7a1 100644
--- a/grails-app/services/au/org/ala/ecodata/MetadataService.groovy
+++ b/grails-app/services/au/org/ala/ecodata/MetadataService.groovy
@@ -12,6 +12,7 @@ import org.apache.poi.ss.usermodel.Sheet
import org.apache.poi.ss.usermodel.Workbook
import org.apache.poi.ss.usermodel.WorkbookFactory
import org.apache.poi.ss.util.CellReference
+import grails.web.mime.MimeType
import java.text.SimpleDateFormat
import java.util.zip.ZipEntry
@@ -707,48 +708,71 @@ class MetadataService {
excelImportService.mapSheet(workbook, config)
}
- List excelWorkbookToMap(InputStream excelWorkbookIn, String activityFormName, Boolean normalise, Integer formVersion = null) {
+ Map excelWorkbookToMap(InputStream excelWorkbookIn, String activityFormName, Boolean normalise, Integer formVersion = null) {
List result = []
+ List errors = []
Workbook workbook = WorkbookFactory.create(excelWorkbookIn)
ActivityForm form = activityFormService.findActivityForm(activityFormName, formVersion)
form?.sections?.each { FormSection section ->
- String sectionName = section.name
- List model = annotatedOutputDataModel(sectionName)
- String sheetName = XlsExporter.sheetName(sectionName)
- Sheet sheet = workbook.getSheet(sheetName)
- def columnMap = excelImportService.getDataHeaders(sheet)
- def config = [
- sheet:sheetName,
- startRow:2,
- columnMap:columnMap
- ]
- List data = excelImportService.mapSheet(workbook, config)
- List normalisedData = []
- if(normalise) {
- data.collect { Map row ->
- def normalisedRow = [:]
- row.each { cell ->
- excelImportService.convertDotNotationToObject(normalisedRow, cell.key, cell.value)
- excelImportService.removeEmptyObjects(normalisedRow)
+ try {
+ String sectionName = section.name
+ List model = annotatedOutputDataModel(sectionName)
+ String sheetName = XlsExporter.sheetName(sectionName)
+ Sheet sheet = workbook.getSheet(sheetName)
+ def columnMap = excelImportService.getDataHeaders(sheet)
+ def config = [
+ sheet:sheetName,
+ startRow:2,
+ columnMap:columnMap
+ ]
+ List data = excelImportService.mapSheet(workbook, config)
+ List normalisedData = []
+ if(normalise) {
+ data.collect { Map row ->
+ def normalisedRow = [:]
+ try {
+ row.each { cell ->
+ excelImportService.convertDotNotationToObject(normalisedRow, cell.key, cell.value)
+ excelImportService.removeEmptyObjects(normalisedRow)
+ }
+
+ addOutputSpeciesIdToSpeciesData(normalisedRow, model)
+ normalisedData << normalisedRow
+ }
+ catch (Exception ex) {
+ errors << [message: messageSource.getMessage('bulkimport.conversionToObjectError', [row.serial].toArray(), '', Locale.default), error: ex.message]
+ }
}
+ }
- addOutputSpeciesIdToSpeciesData(normalisedRow, model)
- normalisedData << normalisedRow
+
+ List rollUpData = []
+ Map groupedBySerial = normalisedData.groupBy {it[OutputUploadTemplateBuilder.SERIAL_NUMBER_DATA]}
+ groupedBySerial.each { key, List rows ->
+ try {
+ rollUpData << rollUpDataIntoSingleElement(rows, model)
+ }
+ catch (Exception ex) {
+ errors << [message: messageSource.getMessage('bulkimport.errorGroupBySerialNumber', [key].toArray(), '', Locale.default), error: ex.message]
+ }
}
- }
- List rollUpData = []
- Map groupedBySerial = normalisedData.groupBy {it[OutputUploadTemplateBuilder.SERIAL_NUMBER_DATA]}
- groupedBySerial.each { key, List rows ->
- rollUpData << rollUpDataIntoSingleElement(rows, model)
+ result.addAll(rollUpData.collect {
+ [[outputName: activityFormName, data: it]]
+ })
+ }
+ catch (Exception ex) {
+ errors << [message: messageSource.getMessage('bulkimport.conversionToFormSectionError', [section.name].toArray(), '', Locale.default), error: ex.message]
}
-
- result.addAll(rollUpData.collect {
- [[outputName: activityFormName, data: it]]
- })
}
- result
+ if (errors) {
+ String errorMessage = errors.collect { it.message + " - " + it.error }.join("
")
+ return [error: errorMessage]
+ }
+ else {
+ return [success: result]
+ }
}
boolean isRowValidNextMemberOfArray(Map row, List models) {
diff --git a/src/test/groovy/au/org/ala/ecodata/MetadataServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/MetadataServiceSpec.groovy
index 7e2a5eae2..72cf817d5 100644
--- a/src/test/groovy/au/org/ala/ecodata/MetadataServiceSpec.groovy
+++ b/src/test/groovy/au/org/ala/ecodata/MetadataServiceSpec.groovy
@@ -270,7 +270,8 @@ class MetadataServiceSpec extends MongoSpec implements ServiceUnitTest