diff --git a/grails-app/controllers/au/org/ala/ecodata/MetadataController.groovy b/grails-app/controllers/au/org/ala/ecodata/MetadataController.groovy index f563c13bd..f0abade58 100644 --- a/grails-app/controllers/au/org/ala/ecodata/MetadataController.groovy +++ b/grails-app/controllers/au/org/ala/ecodata/MetadataController.groovy @@ -9,6 +9,7 @@ import org.springframework.web.multipart.MultipartFile import static au.org.ala.ecodata.Status.DELETED class MetadataController { + static responseFormats = ['json'] def metadataService, activityService, commonService, projectService, webService @@ -235,17 +236,22 @@ class MetadataController { if (file && outputName) { def data = metadataService.excelWorkbookToMap(file.inputStream, outputName, true) + def status = 200 def result - if (!data) { - response.status = 400 + if (data.error) { + status = 500 + data.status = 500 + result = data + } + else if (!data.success) { + status = 400 result = [status:400, error:'No data was found that matched the output description identified by the type parameter, please check the template you used to upload the data. '] } else { - result = [status: 200, data:data] + result = [status: 200, data:data.success] } - render result as JSON - + respond(result, status: status) } else { response.status = 400 diff --git a/grails-app/i18n/messages.properties b/grails-app/i18n/messages.properties index 6ed321f16..606c2b72a 100644 --- a/grails-app/i18n/messages.properties +++ b/grails-app/i18n/messages.properties @@ -309,4 +309,8 @@ projectAcitivity.attribution={0}. ({1}) {2} dataset download. Retrieved from {3} activityForm.invalidIndex=Form template {0} has invalid index fields {1} activityForm.latestVersionIsInDraft = Cannot create a new draft of a form when the current version is a draft -user.userHubs.hubId.unique=Each UserHub must have a unique hubId \ No newline at end of file +user.userHubs.hubId.unique=Each UserHub must have a unique hubId + +bulkimport.conversionToObjectError=Error parsing data into an object in serial number {0} +bulkimport.errorGroupBySerialNumber=Error making nested object from multiple rows in serial number {0} +bulkimport.conversionToFormSectionError=Error parsing data for form section (output) - {0} \ No newline at end of file diff --git a/grails-app/services/au/org/ala/ecodata/MetadataService.groovy b/grails-app/services/au/org/ala/ecodata/MetadataService.groovy index ff5ff114c..b82e6b7a1 100644 --- a/grails-app/services/au/org/ala/ecodata/MetadataService.groovy +++ b/grails-app/services/au/org/ala/ecodata/MetadataService.groovy @@ -12,6 +12,7 @@ import org.apache.poi.ss.usermodel.Sheet import org.apache.poi.ss.usermodel.Workbook import org.apache.poi.ss.usermodel.WorkbookFactory import org.apache.poi.ss.util.CellReference +import grails.web.mime.MimeType import java.text.SimpleDateFormat import java.util.zip.ZipEntry @@ -707,48 +708,71 @@ class MetadataService { excelImportService.mapSheet(workbook, config) } - List excelWorkbookToMap(InputStream excelWorkbookIn, String activityFormName, Boolean normalise, Integer formVersion = null) { + Map excelWorkbookToMap(InputStream excelWorkbookIn, String activityFormName, Boolean normalise, Integer formVersion = null) { List result = [] + List errors = [] Workbook workbook = WorkbookFactory.create(excelWorkbookIn) ActivityForm form = activityFormService.findActivityForm(activityFormName, formVersion) form?.sections?.each { FormSection section -> - String sectionName = section.name - List model = annotatedOutputDataModel(sectionName) - String sheetName = XlsExporter.sheetName(sectionName) - Sheet sheet = workbook.getSheet(sheetName) - def columnMap = excelImportService.getDataHeaders(sheet) - def config = [ - sheet:sheetName, - startRow:2, - columnMap:columnMap - ] - List data = excelImportService.mapSheet(workbook, config) - List normalisedData = [] - if(normalise) { - data.collect { Map row -> - def normalisedRow = [:] - row.each { cell -> - excelImportService.convertDotNotationToObject(normalisedRow, cell.key, cell.value) - excelImportService.removeEmptyObjects(normalisedRow) + try { + String sectionName = section.name + List model = annotatedOutputDataModel(sectionName) + String sheetName = XlsExporter.sheetName(sectionName) + Sheet sheet = workbook.getSheet(sheetName) + def columnMap = excelImportService.getDataHeaders(sheet) + def config = [ + sheet:sheetName, + startRow:2, + columnMap:columnMap + ] + List data = excelImportService.mapSheet(workbook, config) + List normalisedData = [] + if(normalise) { + data.collect { Map row -> + def normalisedRow = [:] + try { + row.each { cell -> + excelImportService.convertDotNotationToObject(normalisedRow, cell.key, cell.value) + excelImportService.removeEmptyObjects(normalisedRow) + } + + addOutputSpeciesIdToSpeciesData(normalisedRow, model) + normalisedData << normalisedRow + } + catch (Exception ex) { + errors << [message: messageSource.getMessage('bulkimport.conversionToObjectError', [row.serial].toArray(), '', Locale.default), error: ex.message] + } } + } - addOutputSpeciesIdToSpeciesData(normalisedRow, model) - normalisedData << normalisedRow + + List rollUpData = [] + Map groupedBySerial = normalisedData.groupBy {it[OutputUploadTemplateBuilder.SERIAL_NUMBER_DATA]} + groupedBySerial.each { key, List rows -> + try { + rollUpData << rollUpDataIntoSingleElement(rows, model) + } + catch (Exception ex) { + errors << [message: messageSource.getMessage('bulkimport.errorGroupBySerialNumber', [key].toArray(), '', Locale.default), error: ex.message] + } } - } - List rollUpData = [] - Map groupedBySerial = normalisedData.groupBy {it[OutputUploadTemplateBuilder.SERIAL_NUMBER_DATA]} - groupedBySerial.each { key, List rows -> - rollUpData << rollUpDataIntoSingleElement(rows, model) + result.addAll(rollUpData.collect { + [[outputName: activityFormName, data: it]] + }) + } + catch (Exception ex) { + errors << [message: messageSource.getMessage('bulkimport.conversionToFormSectionError', [section.name].toArray(), '', Locale.default), error: ex.message] } - - result.addAll(rollUpData.collect { - [[outputName: activityFormName, data: it]] - }) } - result + if (errors) { + String errorMessage = errors.collect { it.message + " - " + it.error }.join("
") + return [error: errorMessage] + } + else { + return [success: result] + } } boolean isRowValidNextMemberOfArray(Map row, List models) { diff --git a/src/test/groovy/au/org/ala/ecodata/MetadataServiceSpec.groovy b/src/test/groovy/au/org/ala/ecodata/MetadataServiceSpec.groovy index 7e2a5eae2..72cf817d5 100644 --- a/src/test/groovy/au/org/ala/ecodata/MetadataServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/ecodata/MetadataServiceSpec.groovy @@ -270,7 +270,8 @@ class MetadataServiceSpec extends MongoSpec implements ServiceUnitTest