diff --git a/build.gradle b/build.gradle index 2bc712646..02c3efac7 100644 --- a/build.gradle +++ b/build.gradle @@ -112,7 +112,7 @@ dependencies { compile "org.nibor.autolink:autolink:0.5.0" compile "com.opencsv:opencsv:3.7" - runtime 'org.cache2k:cache2k-jcache:1.2.0.Final' + runtime 'org.cache2k:cache2k-jcache:2.6.1.Final' testCompile "io.micronaut:micronaut-inject-groovy" testCompile "org.grails:grails-gorm-testing-support" @@ -136,7 +136,7 @@ dependencies { } compile 'au.org.ala:ala-cas-client:3.0.0' - compile ("au.org.ala.names:ala-namematching-client:1.7") { + compile ('au.org.ala.names:ala-namematching-client:1.9-SNAPSHOT') { exclude group: "com.squareup.okhttp3", module: "okhttp" } compile 'au.org.ala.plugins:openapi:1.1.0' diff --git a/grails-app/conf/application.yml b/grails-app/conf/application.yml index 4cf8fbbc3..17997266c 100644 --- a/grails-app/conf/application.yml +++ b/grails-app/conf/application.yml @@ -206,6 +206,8 @@ outboundhttp: namematching: serviceURL: https://namematching-ws.ala.org.au + defaultLoose: false + defaultStyle: STRICT dataCacheConfig: enableJmx: true entryCapacity: 20000 diff --git a/grails-app/controllers/au/org/ala/specieslist/EditorController.groovy b/grails-app/controllers/au/org/ala/specieslist/EditorController.groovy index 24f9a16ed..3092da8ed 100644 --- a/grails-app/controllers/au/org/ala/specieslist/EditorController.groovy +++ b/grails-app/controllers/au/org/ala/specieslist/EditorController.groovy @@ -150,12 +150,14 @@ class EditorController { // check for changed values def keys = SpeciesListKVP.executeQuery("select distinct key from SpeciesListKVP where dataResourceUid= :dataResourceUid", [dataResourceUid: sli.dataResourceUid]) def kvpRemoveList = [] as Set + def changed = false keys.each { key -> def kvp = sli.kvpValues.find { it.key == key } // existing KVP if any if (params[key] != kvp?.value) { log.debug "KVP has been changed: " + params[key] + " VS " + kvp?.value + changed = true def newKvp = SpeciesListKVP.findByDataResourceUidAndKeyAndValue(sli.dataResourceUid, key, params[key]) if (kvp) { @@ -189,14 +191,19 @@ class EditorController { sli.removeFromKvpValues(it) } - //check if rawScientificName has changed + //check if name information has changed if (params.rawScientificName.trim() != sli.rawScientificName.trim()) { log.debug "rawScientificName is different: " + params.rawScientificName + " VS " + sli.rawScientificName sli.rawScientificName = params.rawScientificName + changed = true // lookup guid - helperService.matchNameToSpeciesListItem(sli.rawScientificName, sli) + helperService.matchNameToSpeciesListItem(sli.rawScientificName, sli, sli.mylist) //sli.guid = helperService.findAcceptedLsidByScientificName(sli.rawScientificName)?: helperService.findAcceptedLsidByCommonName(sli.rawScientificName) } + if (changed) { + log.debug "re-matching name for ${params.rawScientificName}" + helperService.matchNameToSpeciesListItem(sli.rawScientificName, sli, sli.mylist) + } if (!sli.validate()) { def message = "Could not update SpeciesListItem: ${sli.rawScientificName} - " + sli.errors.allErrors diff --git a/grails-app/controllers/au/org/ala/specieslist/SpeciesListController.groovy b/grails-app/controllers/au/org/ala/specieslist/SpeciesListController.groovy index 947735166..1fb063d63 100644 --- a/grails-app/controllers/au/org/ala/specieslist/SpeciesListController.groovy +++ b/grails-app/controllers/au/org/ala/specieslist/SpeciesListController.groovy @@ -15,6 +15,7 @@ package au.org.ala.specieslist import au.org.ala.web.AuthService +import au.org.ala.names.ws.api.SearchStyle import com.opencsv.CSVReader import grails.converters.JSON import grails.gorm.transactions.Transactional @@ -30,6 +31,7 @@ class SpeciesListController { private static final String[] ACCEPTED_CONTENT_TYPES = ["text/plain", "text/csv"] HelperService helperService + ColumnMatchingService columnMatchingService AuthService authService BieService bieService BiocacheService biocacheService @@ -164,6 +166,8 @@ class SpeciesListController { formParams.category, formParams.generalisation, formParams.sdsType, + formParams.looseSearch== null || formParams.looseSearch.isEmpty() ? null : Boolean.parseBoolean(formParams.looseSearch), + formParams.searchStyle == null || formParams.searchStyle.isEmpty() ? null : SearchStyle.valueOf(formParams.searchStyle), header.split(","), vocabs) @@ -472,6 +476,7 @@ class SpeciesListController { while (offset < totalRows) { List items List guidBatch = [], sliBatch = [] + Map> batches = new HashMap<>() List searchBatch = new ArrayList() if (id) { items = SpeciesListItem.findAllByDataResourceUid(id, [max: BATCH_SIZE, offset: offset]) @@ -481,10 +486,16 @@ class SpeciesListController { SpeciesListItem.withSession { session -> items.eachWithIndex { SpeciesListItem item, Integer i -> + SpeciesList speciesList = item.mylist + List batch = batches.get(speciesList) + if (batch == null) { + batch = new ArrayList<>(); + batches.put(speciesList, batch) + } String rawName = item.rawScientificName - log.debug i + ". Rematching: " + rawName + log.debug i + ". Rematching: " + rawName + "/" + speciesList.dataResourceUid if (rawName && rawName.length() > 0) { - searchBatch.add(item) + batch.add(item) } else { item.guid = null if (!item.save(flush: true)) { @@ -492,12 +503,13 @@ class SpeciesListController { } } } - - helperService.matchAll(searchBatch) - searchBatch.each {SpeciesListItem item -> - if (item.guid) { - guidBatch.push(item.guid) - sliBatch.push(item) + batches.each { list, batch -> + helperService.matchAll(batch, list) + batch.each {SpeciesListItem item -> + if (item.guid) { + guidBatch.push(item.guid) + sliBatch.push(item) + } } } @@ -522,7 +534,7 @@ class SpeciesListController { private parseDataFromCSV(CSVReader csvReader, String separator) { def rawHeader = csvReader.readNext() log.debug(rawHeader.toList()?.toString()) - def parsedHeader = helperService.parseHeader(rawHeader) ?: helperService.parseData(rawHeader) + def parsedHeader = columnMatchingService.parseHeader(rawHeader) ?: helperService.parseData(rawHeader) def processedHeader = parsedHeader.header log.debug(processedHeader?.toString()) def dataRows = new ArrayList() @@ -531,7 +543,7 @@ class SpeciesListController { dataRows.add(helperService.parseRow(currentLine.toList())) currentLine = csvReader.readNext() } - def nameColumns = helperService.speciesNameColumns + helperService.commonNameColumns + def nameColumns = columnMatchingService.speciesNameMatcher.names + columnMatchingService.commonNameMatcher.names if (processedHeader.find { it == "scientific name" || it == "vernacular name" || it == "common name" || it == "ambiguous name" } && processedHeader.size() > 0) { diff --git a/grails-app/controllers/au/org/ala/specieslist/WebServiceController.groovy b/grails-app/controllers/au/org/ala/specieslist/WebServiceController.groovy index e3fbcb3c1..ec62d3e07 100644 --- a/grails-app/controllers/au/org/ala/specieslist/WebServiceController.groovy +++ b/grails-app/controllers/au/org/ala/specieslist/WebServiceController.groovy @@ -282,12 +282,16 @@ class WebServiceController { dataResourceUid: sl.dataResourceUid, listName : sl.listName, dateCreated : sl.dateCreated, + lastUpdated : sl.lastUpdated, + lastUploaded : sl.lastUploaded, username : sl.username, fullName : sl.getFullName(), itemCount : sl.itemsCount,//SpeciesListItem.countByList(sl) isAuthoritative: (sl.isAuthoritative ?: false), isInvasive : (sl.isInvasive ?: false), - isThreatened : (sl.isThreatened ?: false) + isThreatened : (sl.isThreatened ?: false), + looseSearch : sl.looseSearch, + searchStyle : sl.searchStyle?.toString() ] if (sl.listType) { retValue["listType"] = sl?.listType?.toString() @@ -313,11 +317,13 @@ class WebServiceController { def listCounts = allLists.totalCount def retValue = [listCount: listCounts, sort: params.sort, order: params.order, max: params.max, offset: params.offset, lists : allLists.collect { - [dataResourceUid: it.dataResourceUid, + [ + dataResourceUid: it.dataResourceUid, listName : it.listName, listType : it?.listType?.toString(), dateCreated : it.dateCreated, lastUpdated : it.lastUpdated, + lastUploaded : it.lastUploaded, username : it.username, fullName : it.getFullName(), itemCount : it.itemsCount, @@ -328,9 +334,12 @@ class WebServiceController { sdsType : it.sdsType, isAuthoritative: it.isAuthoritative ?: false, isInvasive : it.isInvasive ?: false, - isThreatened : it.isThreatened ?: false] - }] + isThreatened : it.isThreatened ?: false, + looseSearch : it.looseSearch, + searchStyle : it.searchStyle?.toString() + ] + }] render retValue as JSON } } diff --git a/grails-app/domain/au/org/ala/specieslist/SpeciesList.groovy b/grails-app/domain/au/org/ala/specieslist/SpeciesList.groovy index ee8c064a2..6dab4e7d6 100644 --- a/grails-app/domain/au/org/ala/specieslist/SpeciesList.groovy +++ b/grails-app/domain/au/org/ala/specieslist/SpeciesList.groovy @@ -15,6 +15,8 @@ package au.org.ala.specieslist +import au.org.ala.names.ws.api.SearchStyle + class SpeciesList { def authService @@ -29,6 +31,7 @@ class SpeciesList { String wkt Date dateCreated Date lastUpdated + Date lastUploaded ListType listType Boolean isPrivate Boolean isSDS @@ -42,8 +45,9 @@ class SpeciesList { String generalisation String category String sdsType + Boolean looseSearch // if undefined use the server default + SearchStyle searchStyle // if undefined use the server default String ownerFullName // derived by concatenating the firstName and surname fields - static transients = [ "fullName" ] static hasMany = [items: SpeciesListItem, editors: String] @@ -67,6 +71,9 @@ class SpeciesList { generalisation(nullable: true) authority(nullable: true) sdsType nullable: true + looseSearch nullable: true + searchStyle nullable: true + lastUploaded nullable: true userId nullable: true ownerFullName nullable: true // derived } diff --git a/grails-app/i18n/messages.properties b/grails-app/i18n/messages.properties index a92d317a4..9a0a1ba98 100644 --- a/grails-app/i18n/messages.properties +++ b/grails-app/i18n/messages.properties @@ -95,6 +95,9 @@ speciesList.authority.label=SDS Authority speciesList.category.label=SDS Category speciesList.generalisation.label=SDS Coordinate Generalisation speciesList.sdsType.label=SDS Type +speciesList.looseSearch.label=Loose Name Search +speciesList.searchStyle.label=Name Search Style +speciesList.lastUploaded.label=Date last uploaded upload.heading.hasList=Resubmit to an existing list upload.heading=Upload a list @@ -371,7 +374,9 @@ public.lists.items.header01=Supplied Name public.lists.items.header02=Scientific Name (matched) public.lists.items.header03=Author (matched) public.lists.items.header04=Common Name (matched) -public.lists.items.header05=Image +public.lists.items.header05=Family (matched) +public.lists.items.header06=Kingdom (matched) +public.lists.items.header07=Image generic.lists.label.reload= - Page will be reloaded ... public.lists.items.edit.error=Required field: supplied name cannot be blank diff --git a/grails-app/services/au/org/ala/specieslist/ColumnMatchingService.groovy b/grails-app/services/au/org/ala/specieslist/ColumnMatchingService.groovy new file mode 100644 index 000000000..41904207a --- /dev/null +++ b/grails-app/services/au/org/ala/specieslist/ColumnMatchingService.groovy @@ -0,0 +1,229 @@ +package au.org.ala.specieslist + +import au.org.ala.names.ws.api.NameSearch +import au.org.ala.names.ws.api.SearchStyle +import grails.config.Config +import grails.core.support.GrailsConfigurationAware +import org.apache.commons.lang3.StringUtils + + +/** + * A service that matches column names from/to KVP values + */ +class ColumnMatchingService implements GrailsConfigurationAware { + ColumnMatcher speciesNameMatcher = new ColumnMatcher('rawScientificName', 'name') + ColumnMatcher authorMatcher = new ColumnMatcher('author', 'author') + ColumnMatcher commonNameMatcher = new ColumnMatcher('commonName', 'commonName') + ColumnMatcher ambiguousNameMatcher = new ColumnMatcher('ambiguousName', null) + ColumnMatcher kingdomMatcher = new ColumnMatcher('kingdom', 'kingdom') + ColumnMatcher phylumMatcher = new ColumnMatcher('phylum', 'phylum') + ColumnMatcher classMatcher = new ColumnMatcher('class', 'class') + ColumnMatcher orderMatcher = new ColumnMatcher('order', 'order') + ColumnMatcher familyMatcher = new ColumnMatcher('family', 'family') + ColumnMatcher genusMatcher = new ColumnMatcher('genus', 'genus') + ColumnMatcher rankMatcher = new ColumnMatcher('rank', 'rank') + boolean defaultLoose = false + SearchStyle defaultStyle = SearchStyle.STRICT + + @Override + void setConfiguration(Config configuration) { + this.speciesNameMatcher = new ColumnMatcher('rawScientificName', configuration.getProperty("speciesNameColumns")) + this.authorMatcher = new ColumnMatcher('author', configuration.getProperty("authorColumns")) + this.commonNameMatcher = new ColumnMatcher('commonName', configuration.getProperty("commonNameColumns")) + this.ambiguousNameMatcher = new ColumnMatcher('ambiguousName', configuration.getProperty("ambiguousNameColumns")) + this.kingdomMatcher = new ColumnMatcher('kingdom', configuration.getProperty("kingdomColumns")) + this.phylumMatcher = new ColumnMatcher('phylum', configuration.getProperty("phylumColumns")) + this.classMatcher = new ColumnMatcher('class', configuration.getProperty("classColumns")) + this.orderMatcher = new ColumnMatcher('order', configuration.getProperty("orderColumns")) + this.familyMatcher = new ColumnMatcher('family', configuration.getProperty("familyColumns")) + this.genusMatcher = new ColumnMatcher('genus', configuration.getProperty("genusColumns")) + this.rankMatcher = new ColumnMatcher('rank', configuration.getProperty("rankColumns")) + this.defaultLoose = configuration.getProperty("namematching.defaultLoose", Boolean.class, false) + this.defaultStyle = configuration.getProperty('namematching.defaultStyle', SearchStyle.class, SearchStyle.STRICT) + } + + /** + * Build a name search for a species list item. + *

+ * The scientific name comes from the raw scientifc name. + * All other values come from additional KVP values. + *

+ * + * @param sli The species list item + * @param sl The parent species list (which may not yet be linked to the item) + * @return The name search + */ + NameSearch buildSearch(SpeciesListItem sli, SpeciesList sl) { + boolean loose = sl?.looseSearch ?: this.defaultLoose + SearchStyle style = sl?.searchStyle ?: this.defaultStyle + return NameSearch.builder() + .scientificName(sli.rawScientificName) + .scientificNameAuthorship(this.authorMatcher.get(sli)) + .kingdom(this.kingdomMatcher.get(sli)) + .phylum(this.phylumMatcher.get(sli)) + .clazz(this.classMatcher.get(sli)) + .order(this.orderMatcher.get(sli)) + .family(this.familyMatcher.get(sli)) + .genus(this.genusMatcher.get(sli)) + .rank(this.rankMatcher.get(sli)) + .vernacularName(this.commonNameMatcher.get(sli)) + .loose(loose) + .style(style) + .build(); + } + + /** + * determines what the header should be based on the data supplied + * @param header + */ + def parseData(String[] header) { + def hasName = false + def unknowni = 1 + def headerResponse = header.collect { + if (findAcceptedLsidByScientificName(it)) { + hasName = true + "scientific name" + } else if (findAcceptedLsidByCommonName(it)) { + hasName = true + "vernacular name" + } else { + "UNKNOWN" + (unknowni++) + } + } + [header: headerResponse, nameFound: hasName] + } + + /** + * Check find suitable names from a header + * + * @param header The header + * + * @return The results + */ + def parseHeader(String[] header) { + //first step check to see if scientificname or common name is provided as a header + def hasName = false; + def headerResponse = header.collect { + def search = canonicalName(it) + if (this.speciesNameMatcher.hasName(search)) { + hasName = true + "scientific name" + } else if (this.commonNameMatcher.hasName(search)) { + hasName = true + "vernacular name" + } else if (this.ambiguousNameMatcher.hasName(search)) { + hasName = true + "ambiguous name" + } else { + it + } + } + + headerResponse = parseHeadersCamelCase(headerResponse) + + if (hasName) + [header: headerResponse, nameFound: hasName] + else + null + } + + // specieslist-webapp#50 + protected parseHeadersCamelCase(List header) { + def ret = [] + header.each { String it -> + StringBuilder word = new StringBuilder() + if (Character.isUpperCase(it.codePointAt(0))) { + for (int i = 0; i < it.size(); i++) { + if (Character.isUpperCase(it[i] as char) && i != 0) { + word << " " + } + word << it[i] + } + + ret << word.toString() + } else { + ret << it + } + } + ret + } + + int getSpeciesIndex(Object[] header) { + int idx = header.findIndexOf { this.speciesNameMatcher.hasName(this.canonicalName(it)) } + if (idx < 0) + idx = header.findIndexOf { this.ambiguousNameMatcher.hasName(this.canonicalName(it)) } + if (idx < 0) + idx = header.findIndexOf { this.commonNameMatcher.hasName(this.canonicalName(it)) } + return idx + } + + /** + * Build a map locating various terms in an array for the various columns. + * + * @param header The header + * @return The column-name - column index map + */ + Map getTermAndIndex(Object[] header){ + Map termMap = new HashMap(); + this.speciesNameMatcher.locate(header, termMap) + this.commonNameMatcher.locate(header, termMap) + this.kingdomMatcher.locate(header, termMap) + this.phylumMatcher.locate(header, termMap) + this.classMatcher.locate(header, termMap) + this.orderMatcher.locate(header, termMap) + this.familyMatcher.locate(header, termMap) + this.genusMatcher.locate(header, termMap) + this.rankMatcher.locate(header, termMap) + return termMap + } + + protected String canonicalName(String name) { + return name.toLowerCase().replaceAll('\\s', '') + } + + class ColumnMatcher { + String column; + String[] names + Set matches + + ColumnMatcher(String column, String names) { + this.column = column; + this.names = names?.split(',') ?: [] + this.matches = this.names.inject([], { v, s -> v.add(canonicalName(s)); v}) as Set + } + + /** + * Get a KVP value from an item, based on possible column names. + * + * @param sli The species list item + * @return The first matching value, or null for not found + */ + String get(SpeciesListItem sli) { + for (String name : this.names) { + SpeciesListKVP kvp = sli.kvpValues.find { it.key == name } + String value = kvp ? StringUtils.trimToNull(kvp.value) : null + if (value) + return value + } + return null + } + + /** + * See if we have a specific name. + * @param name A pre-canonicalised name + * @return True if this matches + */ + boolean hasName(String name) { + name = canonicalName(name) + return this.matches.contains(name) + } + + int locate(Object[] header, Map map) { + int idx = header.findIndexOf { hasName(it.toString()) } + if (idx != -1) { + map.put(this.column, idx) + } + return idx + } + } +} \ No newline at end of file diff --git a/grails-app/services/au/org/ala/specieslist/HelperService.groovy b/grails-app/services/au/org/ala/specieslist/HelperService.groovy index b8d42e976..5d8757248 100644 --- a/grails-app/services/au/org/ala/specieslist/HelperService.groovy +++ b/grails-app/services/au/org/ala/specieslist/HelperService.groovy @@ -16,6 +16,7 @@ package au.org.ala.specieslist import au.org.ala.names.ws.api.NameUsageMatch +import au.org.ala.names.ws.api.SearchStyle import com.opencsv.CSVReader import grails.gorm.transactions.Transactional import groovyx.net.http.ContentType @@ -37,16 +38,6 @@ import javax.annotation.PostConstruct @Transactional class HelperService { - private static final String COMMON_NAME = "commonName" - private static final String KINGDOM = "kingdom" - private static final String RAW_SCIENTIFIC_NAME = "rawScientificName" - private static final String PHYLUM = "phylum" - private static final String CLASS = "class" - private static final String ORDER = "order" - private static final String FAMILY = "family" - private static final String GENUS = "genus" - private static final String RANK = "rank" - MessageSource messageSource def grailsApplication @@ -57,19 +48,9 @@ class HelperService { NameExplorerService nameExplorerService - Integer BATCH_SIZE - - String[] speciesNameColumns = [] - String[] commonNameColumns = [] - String[] ambiguousNameColumns = [] - String[] kingdomColumns = [] - String[] phylumColumns = [] - String[] classColumns = [] - String[] orderColumns = [] - String[] familyColumns = [] - String[] genusColumns = [] - String[] rankColumns = [] + ColumnMatchingService columnMatchingService + Integer BATCH_SIZE // Only permit URLs for added safety private final LinkExtractor extractor = LinkExtractor.builder().linkTypes(EnumSet.of(LinkType.URL)).build() @@ -77,27 +58,7 @@ class HelperService { @PostConstruct init(){ BATCH_SIZE = Integer.parseInt((grailsApplication?.config?.batchSize?:200).toString()) - speciesNameColumns = grailsApplication?.config?.speciesNameColumns ? - grailsApplication?.config?.speciesNameColumns?.split(',') : [] - commonNameColumns = grailsApplication?.config?.commonNameColumns ? - grailsApplication?.config?.commonNameColumns?.split(',') : [] - ambiguousNameColumns = grailsApplication?.config?.ambiguousNameColumns ? - grailsApplication?.config?.ambiguousNameColumns?.split(',') : [] - kingdomColumns = grailsApplication?.config?.kingdomColumns ? - grailsApplication?.config?.kingdomColumns?.split(',') : [] - phylumColumns = grailsApplication?.config?.phylumColumns ? - grailsApplication?.config?.phylumColumns?.split(',') : [] - classColumns = grailsApplication?.config?.classColumns ? - grailsApplication?.config?.classColumns?.split(',') : [] - orderColumns = grailsApplication?.config?.orderColumns ? - grailsApplication?.config?.orderColumns?.split(',') : [] - familyColumns = grailsApplication?.config?.familyColumns ? - grailsApplication?.config?.familyColumns?.split(',') : [] - genusColumns = grailsApplication?.config?.genusColumns ? - grailsApplication?.config?.genusColumns?.split(',') : [] - rankColumns = grailsApplication?.config?.rankColumns ? - grailsApplication?.config?.rankColumns?.split(',') : [] - } + } /** * Adds a data resource to the collectory for this species list @@ -276,56 +237,6 @@ class HelperService { [header: headerResponse, nameFound: hasName] } - def parseHeader(String[] header) { - //first step check to see if scientificname or common name is provided as a header - def hasName = false; - def headerResponse = header.collect { - def search = it.toLowerCase().replaceAll(" ", "") - if (speciesNameColumns.contains(search)) { - hasName = true - "scientific name" - } else if (commonNameColumns.contains(search)) { - hasName = true - "vernacular name" - } else if (ambiguousNameColumns.contains(search)) { - hasName = true - "ambiguous name" - } else { - it - } - } - - headerResponse = parseHeadersCamelCase(headerResponse) - - if (hasName) - [header: headerResponse, nameFound: hasName] - else - null - } - - // specieslist-webapp#50 - def parseHeadersCamelCase(List header) { - def ret = [] - header.each {String it -> - StringBuilder word = new StringBuilder() - if (Character.isUpperCase(it.codePointAt(0))) { - for (int i = 0; i < it.size(); i++) { - if (Character.isUpperCase(it[i] as char) && i != 0) { - word << " " - } - word << it[i] - } - - ret << word.toString() - } - else { - ret << it - } - } - - ret - } - def parseRow(List row) { def ret = [] @@ -359,35 +270,6 @@ class HelperService { ret } - def getSpeciesIndex(Object[] header) { - int idx = header.findIndexOf { speciesNameColumns.contains(it.toString().toLowerCase().replaceAll(" ", "")) } - if (idx < 0) - idx = header.findIndexOf { commonNameColumns.contains(it.toString().toLowerCase().replaceAll(" ", "")) } - return idx - } - - private Map getTermAndIndex(Object[] header){ - Map termMap = new HashMap(); - locateValues(termMap, header, speciesNameColumns, RAW_SCIENTIFIC_NAME) - locateValues(termMap, header, commonNameColumns, COMMON_NAME) - locateValues(termMap, header, kingdomColumns, KINGDOM) - locateValues(termMap, header, phylumColumns, PHYLUM) - locateValues(termMap, header, classColumns, CLASS) - locateValues(termMap, header, orderColumns, ORDER) - locateValues(termMap, header, familyColumns, FAMILY) - locateValues(termMap, header, genusColumns, GENUS) - locateValues(termMap, header, rankColumns, RANK) - - return termMap - } - - private void locateValues(Map map, Object[] header, String[] cols, String term) { - int idx = header.findIndexOf { cols.contains(it.toString().toLowerCase().replaceAll(" ","")) } - if (idx != -1) { - map.put(term, idx) - } - } - private boolean hasValidData(Map map, String [] nextLine) { boolean result = false map.each { key, value -> @@ -460,6 +342,7 @@ class HelperService { } else { throw new UnsupportedOperationException("Unsupported data structure") } + speciesList.lastUploaded = new Date() if (!speciesList.validate()) { log.error(speciesList.errors.allErrors?.toString()) @@ -485,7 +368,7 @@ class HelperService { List guidList = [] items.eachWithIndex { item, i -> SpeciesListItem sli = new SpeciesListItem(dataResourceUid: druid, rawScientificName: item, itemOrder: i) - matchNameToSpeciesListItem(sli.rawScientificName, sli) + matchNameToSpeciesListItem(sli.rawScientificName, sli, speciesList) speciesList.addToItems(sli) guidList.push (sli.guid) } @@ -506,7 +389,6 @@ class HelperService { items.eachWithIndex { item, i -> SpeciesListItem sli = new SpeciesListItem(dataResourceUid: druid, rawScientificName: item.itemName, itemOrder: i) - matchNameToSpeciesListItem(sli.rawScientificName, sli) item.kvpValues?.eachWithIndex { k, j -> SpeciesListKVP kvp = new SpeciesListKVP(value: k.value, key: k.key, itemOrder: j, dataResourceUid: @@ -514,6 +396,7 @@ class HelperService { sli.addToKvpValues(kvp) kvpMap[k.key] = k.value } + matchNameToSpeciesListItem(sli.rawScientificName, sli, speciesList) speciesList.addToItems(sli) @@ -524,7 +407,7 @@ class HelperService { def loadSpeciesListFromCSV(CSVReader reader, druid, listname, ListType listType, description, listUrl, listWkt, Boolean isBIE, Boolean isSDS, Boolean isPrivate, String region, String authority, String category, - String generalisation, String sdsType, String[] header, Map vocabs) { + String generalisation, String sdsType, Boolean looseSearch, SearchStyle searchStyle, String [] header, Map vocabs) { log.debug("Loading species list " + druid + " " + listname + " " + description + " " + listUrl + " " + header + " " + vocabs) def kvpmap = [:] addVocab(druid,vocabs,kvpmap) @@ -554,9 +437,12 @@ class HelperService { sl.isAuthoritative = false // default all new lists to isAuthoritative = false: it is an admin task to determine whether a list is authoritative or not sl.isInvasive = false sl.isThreatened = false + sl.looseSearch = looseSearch + sl.searchStyle = searchStyle + sl.lastUploaded = new Date() String [] nextLine boolean checkedHeader = false - Map termIdx = getTermAndIndex(header) + Map termIdx = columnMatchingService.getTermAndIndex(header) int itemCount = 0 int totalCount = 0 while ((nextLine = reader.readNext()) != null) { @@ -564,14 +450,14 @@ class HelperService { if(!checkedHeader){ checkedHeader = true // only read next line if current line is a header line - if(getTermAndIndex(nextLine).size() > 0) { + if(columnMatchingService.getTermAndIndex(nextLine).size() > 0) { nextLine = reader.readNext() } } if(nextLine.length > 0 && termIdx.size() > 0 && hasValidData(termIdx, nextLine)){ itemCount++ - sl.addToItems(insertSpeciesItem(nextLine, druid, termIdx, header, kvpmap, itemCount)) + sl.addToItems(insertSpeciesItem(nextLine, druid, termIdx, header, kvpmap, itemCount, sl)) } } @@ -591,7 +477,7 @@ class HelperService { CSVReader reader = new CSVReader(new FileReader(filename),',' as char) header = header ?: reader.readNext() - int speciesValueIdx = getSpeciesIndex(header) + int speciesValueIdx = columnMatchingService.getSpeciesIndex(header) int count =0 String [] nextLine def kvpmap =[:] @@ -603,9 +489,10 @@ class HelperService { sl.username = localAuthService.email() sl.firstName = localAuthService.firstname() sl.surname = localAuthService.surname() + sl.lastUploaded = new Date() while ((nextLine = reader.readNext()) != null) { if(org.apache.commons.lang.StringUtils.isNotBlank(nextLine)){ - sl.addToItems(insertSpeciesItem(nextLine, druid, speciesValueIdx, header,kvpmap)) + sl.addToItems(insertSpeciesItem(nextLine, druid, speciesValueIdx, header,kvpmap, sl)) count++ } @@ -617,7 +504,7 @@ class HelperService { sl.save() } - def insertSpeciesItem(String[] values, druid, int speciesIdx, Object[] header, map, int order){ + def insertSpeciesItem(String[] values, druid, int speciesIdx, Object[] header, map, int order, SpeciesList sl){ values = parseRow(values as List) log.debug("Inserting " + values.toArrayString()) @@ -625,10 +512,7 @@ class HelperService { sli.dataResourceUid =druid sli.rawScientificName = speciesIdx > -1 ? values[speciesIdx] : null sli.itemOrder = order - //lookup the raw - //sli.guid = findAcceptedLsidByScientificName(sli.rawScientificName)?: findAcceptedLsidByCommonName(sli.rawScientificName) - matchNameToSpeciesListItem(sli.rawScientificName, sli) - int i = 0 + int i = 0 header.each { if(i != speciesIdx && values.length > i && values[i]?.trim()){ SpeciesListKVP kvp = map.get(it.toString()+"|"+values[i], new SpeciesListKVP(key: it.toString(), value: values[i], dataResourceUid: druid)) @@ -639,20 +523,19 @@ class HelperService { } i++ } - + matchNameToSpeciesListItem(sli.rawScientificName, sli, sl) sli } - def insertSpeciesItem(String[] values, String druid, Map termIndex, Object[] header, Map map, int order){ + def insertSpeciesItem(String[] values, String druid, Map termIndex, Object[] header, Map map, int order, SpeciesList sl){ values = parseRow(values as List) log.debug("Inserting " + values.toArrayString()) SpeciesListItem sli = new SpeciesListItem() sli.dataResourceUid = druid - sli.rawScientificName = termIndex.containsKey(RAW_SCIENTIFIC_NAME) ? values[termIndex[RAW_SCIENTIFIC_NAME]] : null + sli.rawScientificName = termIndex.containsKey(QueryService.RAW_SCIENTIFIC_NAME) ? values[termIndex[QueryService.RAW_SCIENTIFIC_NAME]] : null sli.itemOrder = order - matchValuesToSpeciesListItem(values, termIndex, sli) int i = 0 header.each { if(!termIndex.containsValue(i) && values.length > i && values[i]?.trim()){ @@ -664,48 +547,52 @@ class HelperService { } i++ } + matchNameToSpeciesListItem(sli.rawScientificName, sli, sl) sli } - def matchNameToSpeciesListItem(String name, SpeciesListItem sli){ - //includes matchedName search for rematching if nameSearcher lsids change. - NameUsageMatch nameUsageMatch = findAcceptedConceptByScientificName(sli.rawScientificName) ?: - findAcceptedConceptByCommonName(sli.rawScientificName) ?: - findAcceptedConceptByLSID(sli.rawScientificName) ?: - findAcceptedConceptByNameFamily(sli.matchedName, sli.family) - if(nameUsageMatch){ - sli.guid = nameUsageMatch.getTaxonConceptID() - sli.family = nameUsageMatch.getFamily() - sli.matchedName = nameUsageMatch.getScientificName() - sli.author = nameUsageMatch.getScientificNameAuthorship() - sli.commonName = nameUsageMatch.getVernacularName() - sli.kingdom = nameUsageMatch.getKingdom() + def matchNameToSpeciesListItem(String name, SpeciesListItem sli, SpeciesList sl){ + // First match using all available data + NameUsageMatch match = nameExplorerService.find(sli, sl) + if (!match || !match.success) { + match = nameExplorerService.searchForRecordByCommonName(sli.rawScientificName) } - } - - def rematchToSpeciesListItem(SpeciesListItem sli){ - NameUsageMatch nameUsageMatch = nameExplorerService.searchForRecordByTerms(sli.rawScientificName, sli.commonName, - sli.kingdom, null, null, null, sli.family, null, null) - if(nameUsageMatch){ - sli.guid = nameUsageMatch.getTaxonConceptID() - sli.family = nameUsageMatch.getFamily() - sli.matchedName = nameUsageMatch.getScientificName() - sli.author = nameUsageMatch.getScientificNameAuthorship() - sli.commonName = nameUsageMatch.getVernacularName() - sli.kingdom = nameUsageMatch.getKingdom() + if (!match || !match.success) { + match = nameExplorerService.searchForRecordByLsid(sli.rawScientificName) + } + if(match && match.success){ + sli.guid = match.getTaxonConceptID() + sli.matchedName = match.getScientificName() + sli.author = match.getScientificNameAuthorship() + sli.commonName = match.getVernacularName() + sli.family = match.getFamily() + sli.kingdom = match.getKingdom() + } else { + sli.guid = null + sli.matchedName = null + sli.author = null + sli.commonName = null + sli.family = null + sli.kingdom = null } } - void matchAll(List searchBatch) { - List matches = nameExplorerService.findAll(searchBatch); - matches.eachWithIndex { NameUsageMatch match, Integer index -> + void matchAll(List searchBatch, SpeciesList speciesList) { + List matches = nameExplorerService.findAll(searchBatch, speciesList); + matches.eachWithIndex { NameUsageMatch match, Integer index -> SpeciesListItem sli = searchBatch[index] + if (!match.success) { + match = nameExplorerService.searchForRecordByCommonName(sli.rawScientificName) + } + if (!match.success) { + match = nameExplorerService.searchForRecordByLsid(sli.rawScientificName) + } if (match && match.success) { sli.guid = match.getTaxonConceptID() - sli.family = match.getFamily() sli.matchedName = match.getScientificName() sli.author = match.getScientificNameAuthorship() sli.commonName = match.getVernacularName() + sli.family = match.getFamily() sli.kingdom = match.getKingdom() } else { log.info("Unable to match species list item - ${sli.rawScientificName}") @@ -713,29 +600,6 @@ class HelperService { } } - def matchValuesToSpeciesListItem(String[] values, Map termIndex, SpeciesListItem sli){ - String rawScientificName = termIndex.containsKey(RAW_SCIENTIFIC_NAME) ? values[termIndex[RAW_SCIENTIFIC_NAME]] : null - String family = termIndex.containsKey(FAMILY) ? values[termIndex[FAMILY]] :null - String commonName = termIndex.containsKey(COMMON_NAME) ? values[termIndex[COMMON_NAME]] :null - String kingdom = termIndex.containsKey(KINGDOM) ? values[termIndex[KINGDOM]] :null - String phylum = termIndex.containsKey(PHYLUM) ? values[termIndex[PHYLUM]] :null - String clazz = termIndex.containsKey(CLASS) ? values[termIndex[CLASS]] :null - String order = termIndex.containsKey(ORDER) ? values[termIndex[ORDER]] :null - String genus = termIndex.containsKey(GENUS) ? values[termIndex[GENUS]] :null - String rank = termIndex.containsKey(RANK) ? values[termIndex[RANK]] :null - - NameUsageMatch nameUsageMatch = nameExplorerService.searchForRecordByTerms(rawScientificName, commonName, - kingdom, phylum, clazz, order, family, genus, rank) - if(nameUsageMatch){ - sli.guid = nameUsageMatch.getTaxonConceptID() - sli.family = nameUsageMatch.getFamily() - sli.matchedName = nameUsageMatch.getScientificName() - sli.author = nameUsageMatch.getScientificNameAuthorship() - sli.commonName = nameUsageMatch.getVernacularName() - sli.kingdom = nameUsageMatch.getKingdom() - } - } - def findAcceptedLsidByCommonName(commonName) { String lsid = null try { @@ -756,50 +620,6 @@ class HelperService { lsid } - def findAcceptedConceptByLSID(lsid) { - NameUsageMatch record - try { - record = nameExplorerService.searchForRecordByLsid(lsid) - } - catch (Exception e) { - log.error(e.getMessage()) - } - record - } - - def findAcceptedConceptByNameFamily(String scientificName, String family) { - NameUsageMatch record - try { - record = nameExplorerService.searchForRecordByNameFamily(scientificName, family) - } - catch (Exception e) { - log.error(e.getMessage()) - } - record - } - - def findAcceptedConceptByScientificName(scientificName) { - NameUsageMatch record - try { - record = nameExplorerService.searchForRecordByScientificName(scientificName) - } - catch (Exception e) { - log.error(e.getMessage()) - } - record - } - - def findAcceptedConceptByCommonName(commonName) { - NameUsageMatch record - try { - record = nameExplorerService.searchForRecordByCommonName(commonName) - } - catch (Exception e) { - log.error(e.getMessage()) - } - record - } - // JSON response is returned as the unconverted model with the appropriate // content-type. The JSON conversion is handled in the filter. This allows // for universal JSONP support. @@ -845,7 +665,7 @@ class HelperService { def keys = SpeciesListKVP.executeQuery("select distinct key from SpeciesListKVP where dataResourceUid=:dataResourceUid", [dataResourceUid: sl.dataResourceUid]) log.debug "keys = " + keys def sli = new SpeciesListItem(dataResourceUid: sl.dataResourceUid, rawScientificName: params.rawScientificName, itemOrder: sl.items.size() + 1) - matchNameToSpeciesListItem(sli.rawScientificName, sli) + matchNameToSpeciesListItem(sli.rawScientificName, sli, sl) keys.each { key -> log.debug "key: " + key + " has value: " + params[key] diff --git a/grails-app/services/au/org/ala/specieslist/NameExplorerService.groovy b/grails-app/services/au/org/ala/specieslist/NameExplorerService.groovy index c98341c90..2fd637e99 100644 --- a/grails-app/services/au/org/ala/specieslist/NameExplorerService.groovy +++ b/grails-app/services/au/org/ala/specieslist/NameExplorerService.groovy @@ -32,17 +32,17 @@ import org.apache.commons.lang.StringUtils class NameExplorerService implements GrailsConfigurationAware { private NameMatchService alaNameUsageMatchServiceClient - def grailsApplication + ColumnMatchingService columnMatchingService @Override void setConfiguration(Config config) { DataCacheConfiguration dataCacheConfig = DataCacheConfiguration.builder() - .entryCapacity(grailsApplication.config.getProperty("namematching.dataCacheConfig.entryCapacity", Integer, 20000)) - .enableJmx(grailsApplication.config.getProperty("namematching.dataCacheConfig.enableJmx", Boolean, false)) - .eternal(grailsApplication.config.getProperty("namematching.dataCacheConfig.eternal", Boolean, false)) - .keepDataAfterExpired(grailsApplication.config.getProperty("namematching.dataCacheConfig.keepDataAfterExpired", Boolean, false)) - .permitNullValues(grailsApplication.config.getProperty("namematching.dataCacheConfig.permitNullValues", Boolean, false)) - .suppressExceptions(grailsApplication.config.getProperty("namematching.dataCacheConfig.suppressExceptions", Boolean, false)) + .entryCapacity(config.getProperty("namematching.dataCacheConfig.entryCapacity", Integer, 20000)) + .enableJmx(config.getProperty("namematching.dataCacheConfig.enableJmx", Boolean, false)) + .eternal(config.getProperty("namematching.dataCacheConfig.eternal", Boolean, false)) + .keepDataAfterExpired(config.getProperty("namematching.dataCacheConfig.keepDataAfterExpired", Boolean, false)) + .permitNullValues(config.getProperty("namematching.dataCacheConfig.permitNullValues", Boolean, false)) + .suppressExceptions(config.getProperty("namematching.dataCacheConfig.suppressExceptions", Boolean, false)) .build() URL service = new URL(config.getProperty("namematching.serviceURL")) @@ -55,23 +55,22 @@ class NameExplorerService implements GrailsConfigurationAware { /** * Find a NameUsageMatch by a NameSearch - * @param search NameUsageMatch + * @param sli The species list item + * @param sl The parent species list (which may not yet be linked to the item) * @return NameSearch */ - NameUsageMatch find(NameSearch search) { - return alaNameUsageMatchServiceClient.match(search) + NameUsageMatch find(SpeciesListItem sli, SpeciesList sl) { + return alaNameUsageMatchServiceClient.match(columnMatchingService.buildSearch(sli, sl)) } /** * Find NameUsageMatch for given list items * @param items list of SpeciesListItem + * @param sl The parent species list (which may not yet be linked to the item) * @return list of NameUsageMatch, in the same order as the items passed in */ - List findAll(List items){ - List searches = new ArrayList<>(); - items.eachWithIndex { SpeciesListItem item, Integer i -> - searches.add(i, buildNameSearch(item)) - } + List findAll(List items, SpeciesList sl){ + List searches = items.collect { sli -> columnMatchingService.buildSearch(sli, sl)} return alaNameUsageMatchServiceClient.matchAll(searches) } @@ -94,7 +93,7 @@ class NameExplorerService implements GrailsConfigurationAware { NameSearch.NameSearchBuilder builder = new NameSearch.NameSearchBuilder() builder.scientificName = StringUtils.trimToNull(scientificName) - NameUsageMatch result = find(builder.build()) + NameUsageMatch result = alaNameUsageMatchServiceClient.match(builder.build()) return result.success? result.taxonConceptID : null } @@ -119,7 +118,7 @@ class NameExplorerService implements GrailsConfigurationAware { builder.scientificName = StringUtils.trimToNull(scientificName) builder.family = StringUtils.trimToNull(family) - NameUsageMatch result = find(builder.build()) + NameUsageMatch result = alaNameUsageMatchServiceClient.match(builder.build()) return result.success? result : null } @@ -132,7 +131,7 @@ class NameExplorerService implements GrailsConfigurationAware { NameSearch.NameSearchBuilder builder = new NameSearch.NameSearchBuilder() builder.scientificName = StringUtils.trimToNull(scientificName) - NameUsageMatch result = find(builder.build()) + NameUsageMatch result = alaNameUsageMatchServiceClient.match(builder.build()) return result.success? result : null } @@ -145,7 +144,7 @@ class NameExplorerService implements GrailsConfigurationAware { NameSearch.NameSearchBuilder builder = new NameSearch.NameSearchBuilder() builder.vernacularName = StringUtils.trimToNull(commonName) - NameUsageMatch result = find(builder.build()) + NameUsageMatch result = alaNameUsageMatchServiceClient.match(builder.build()) return result.success? result : null } @@ -200,24 +199,8 @@ class NameExplorerService implements GrailsConfigurationAware { if (rank) { builder.rank = StringUtils.trimToNull(rank) } - NameUsageMatch result = find(builder.build()) + NameUsageMatch result = alaNameUsageMatchServiceClient.match(builder.build()) return result.success? result : null } - private NameSearch buildNameSearch(SpeciesListItem sli){ - NameSearch.NameSearchBuilder builder = new NameSearch.NameSearchBuilder() - if (sli.rawScientificName) { - builder.scientificName = StringUtils.trimToNull(sli.rawScientificName) - } - if (sli.commonName) { - builder.vernacularName = StringUtils.trimToNull(sli.commonName) - } - if (sli.kingdom) { - builder.kingdom = StringUtils.trimToNull(sli.kingdom) - } - if (sli.family) { - builder.family = StringUtils.trimToNull(sli.family) - } - return builder.loose(true).build() - } -} + } diff --git a/grails-app/views/speciesList/upload.gsp b/grails-app/views/speciesList/upload.gsp index d7f97b5c1..78b9df6e8 100644 --- a/grails-app/views/speciesList/upload.gsp +++ b/grails-app/views/speciesList/upload.gsp @@ -1,3 +1,4 @@ +<%@ page import="au.org.ala.names.ws.api.SearchStyle" %> %{-- - Copyright (C) 2012 Atlas of Living Australia - All Rights Reserved. @@ -181,6 +182,8 @@ map['sdsType'] = $('#sdsType').val(); } } + map['looseSearch'] = $('#looseSearch').val(); + map['searchStyle'] = $('#searchStyle').val(); //console.log("The map: ",map); $('#recognisedDataDiv').hide(); $('#uploadDiv').hide(); @@ -442,7 +445,18 @@ ${list?.wkt} - + + + + + + + + + + + + diff --git a/grails-app/views/speciesListItem/list.gsp b/grails-app/views/speciesListItem/list.gsp index 8244dc4f5..c141c5ea1 100644 --- a/grails-app/views/speciesListItem/list.gsp +++ b/grails-app/views/speciesListItem/list.gsp @@ -479,10 +479,13 @@
${message(code: 'speciesList.dateCreated.label', default: 'Date submitted')}
+ date="${speciesList.dateCreated}"/>
${message(code: 'speciesList.lastUpdated.label', default: 'Date updated')}
+ date="${speciesList.lastUpdated}"/> +
${message(code: 'speciesList.lastUploaded.label', default: 'Date last loaded')}
+
${message(code: 'speciesList.isPrivate.label', default: 'Is private')}
${message(code: 'speciesList.isBIE.label', default: 'Included in BIE')}
@@ -519,6 +522,10 @@
${message(code: 'speciesList.editors.label', default: 'List editors')}
${speciesList.editors.collect { sl.getFullNameForUserId(userId: it) }?.join(", ")}
+
${message(code: 'speciesList.looseSearch.label', default: 'Loose search')}
+
+
${message(code: 'speciesList.searchStyle.label', default: 'Search style')}
+
${speciesList.searchStyle}
${message(code: 'speciesList.metadata.label', default: 'Metadata link')}
${grailsApplication.config.collectory.baseURL}/public/show/${speciesList.dataResourceUid}
@@ -716,6 +723,18 @@ +
+ +
+ +
+
+
+ +
+ +
+
@@ -905,11 +924,15 @@ params="${[fq: fqs]}"> - ${message(code:'public.lists.items.header05', default:'Image')} + ${message(code:'public.lists.items.header07', default:'Image')} + + ${key} @@ -964,6 +987,8 @@ ${result.author} ${result.commonName} + ${result.family} + ${result.kingdom} diff --git a/src/test/groovy/au/org/ala/specieslist/controller/SpeciesListControllerSpec.groovy b/src/test/groovy/au/org/ala/specieslist/controller/SpeciesListControllerSpec.groovy index 2ea294e3a..531c8cab4 100644 --- a/src/test/groovy/au/org/ala/specieslist/controller/SpeciesListControllerSpec.groovy +++ b/src/test/groovy/au/org/ala/specieslist/controller/SpeciesListControllerSpec.groovy @@ -15,6 +15,7 @@ package au.org.ala.specieslist.controller +import au.org.ala.specieslist.ColumnMatchingService import au.org.ala.specieslist.HelperService import au.org.ala.specieslist.SpeciesListController import grails.testing.gorm.DataTest @@ -36,6 +37,7 @@ class SpeciesListControllerSpec extends Specification implements ControllerUnitT def setup() { mockMultipartRequest.getFile(_) >> mockFile controller.helperService = new HelperService() + controller.columnMatchingService = new ColumnMatchingService() // controller.helperService.transactionManager = transactionManager } diff --git a/src/test/groovy/au/org/ala/specieslist/service/ColumnMatchingServiceSpec.groovy b/src/test/groovy/au/org/ala/specieslist/service/ColumnMatchingServiceSpec.groovy new file mode 100644 index 000000000..c2de4ecf1 --- /dev/null +++ b/src/test/groovy/au/org/ala/specieslist/service/ColumnMatchingServiceSpec.groovy @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2022 Atlas of Living Australia + * All Rights Reserved. + * + * The contents of this file are subject to the Mozilla Public + * License Version 1.1 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of + * the License at http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS + * IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or + * implied. See the License for the specific language governing + * rights and limitations under the License. + */ + +package au.org.ala.specieslist.service + +import au.org.ala.names.ws.api.NameUsageMatch +import au.org.ala.specieslist.* +import au.org.ala.web.AuthService +import com.opencsv.CSVReader +import grails.testing.gorm.DataTest +import grails.testing.services.ServiceUnitTest +import org.grails.web.json.JSONObject +import spock.lang.Specification +import spock.lang.Unroll + +@Unroll +class ColumnMatchingServiceSpec extends Specification implements ServiceUnitTest, DataTest { + + def columnMatchingService = new ColumnMatchingService() + + void setupSpec() { + mockDomains(SpeciesListItem, SpeciesList, SpeciesListKVP) + } + + def setup() { + grailsApplication.config.commonNameColumns="commonname,vernacularname" + grailsApplication.config.ambiguousNameColumns="name" + grailsApplication.config.speciesNameColumns = "scientificname,suppliedname,taxonname,species" + columnMatchingService.setConfiguration(grailsApplication.config) + } + + + def "camel case column names should be split by spaces before each uppercase character"() { + when: + def result = columnMatchingService.parseHeader( + ["species", "AnyReallyLongCamelCaseHeaderName", "ÖsterreichName", "conservationCode"] as String[]) + + then: + assert result?.header?.contains("scientific name") + assert result?.header?.contains("Any Really Long Camel Case Header Name") + assert result?.header?.contains("Österreich Name"); + assert result?.header?.contains("conservationCode") + } + +} diff --git a/src/test/groovy/au/org/ala/specieslist/service/HelperServiceSpec.groovy b/src/test/groovy/au/org/ala/specieslist/service/HelperServiceSpec.groovy index c91742ad5..6fac9917a 100644 --- a/src/test/groovy/au/org/ala/specieslist/service/HelperServiceSpec.groovy +++ b/src/test/groovy/au/org/ala/specieslist/service/HelperServiceSpec.groovy @@ -16,6 +16,7 @@ package au.org.ala.specieslist.service import au.org.ala.names.ws.api.NameUsageMatch +import au.org.ala.names.ws.api.SearchStyle import au.org.ala.specieslist.* import au.org.ala.web.AuthService import au.org.ala.web.UserDetails @@ -30,6 +31,8 @@ import spock.lang.Unroll class HelperServiceSpec extends Specification implements ServiceUnitTest, DataTest { def helperService = new HelperService() + def columnMatchingService = new ColumnMatchingService() + def nameExplorerService = Mock(NameExplorerService) void setupSpec() { mockDomains(SpeciesListItem, SpeciesList, SpeciesListKVP) @@ -40,8 +43,24 @@ class HelperServiceSpec extends Specification implements ServiceUnitTest>> result } def "addDataResourceForList should return a dummy url when collectory.enableSync is not true - #item"() { @@ -72,7 +91,9 @@ class HelperServiceSpec extends Specification implements ServiceUnitTest>> result + nameExplorerService.searchForRecordByTerms(*_) >>> result helperService.setLocalAuthService(Mock(LocalAuthService)) helperService.setAuthService(Mock(AuthService)) @@ -149,7 +190,7 @@ class HelperServiceSpec extends Specification implements ServiceUnitTest