Allow strict name matching of lists

build.gradle

@@ -112,7 +112,7 @@ dependencies {
     compile "org.nibor.autolink:autolink:0.5.0"
     compile "com.opencsv:opencsv:3.7"
 
-    runtime 'org.cache2k:cache2k-jcache:1.2.0.Final'
+    runtime 'org.cache2k:cache2k-jcache:2.6.1.Final'
 
     testCompile "io.micronaut:micronaut-inject-groovy"
     testCompile "org.grails:grails-gorm-testing-support"
@@ -136,7 +136,7 @@ dependencies {
     }
 
     compile 'au.org.ala:ala-cas-client:3.0.0'
-    compile ("au.org.ala.names:ala-namematching-client:1.7") {
+    compile ('au.org.ala.names:ala-namematching-client:1.9-SNAPSHOT') {
         exclude group: "com.squareup.okhttp3", module: "okhttp"
     }
     compile 'au.org.ala.plugins:openapi:1.1.0'

grails-app/conf/application.yml

@@ -206,6 +206,8 @@ outboundhttp:
 
 namematching:
     serviceURL: https://namematching-ws.ala.org.au
+    defaultLoose: false
+    defaultStyle: STRICT
     dataCacheConfig:
         enableJmx: true
         entryCapacity: 20000

grails-app/controllers/au/org/ala/specieslist/EditorController.groovy

@@ -150,12 +150,14 @@ class EditorController {
             // check for changed values
             def keys = SpeciesListKVP.executeQuery("select distinct key from SpeciesListKVP where dataResourceUid= :dataResourceUid", [dataResourceUid: sli.dataResourceUid])
             def kvpRemoveList = [] as Set
+            def changed = false
 
             keys.each { key ->
                 def kvp = sli.kvpValues.find { it.key == key } // existing KVP if any
 
                 if (params[key] != kvp?.value) {
                     log.debug "KVP has been changed: " + params[key] + " VS " + kvp?.value
+                    changed = true
                     def newKvp = SpeciesListKVP.findByDataResourceUidAndKeyAndValue(sli.dataResourceUid, key, params[key])
 
                     if (kvp) {
@@ -189,14 +191,19 @@ class EditorController {
                 sli.removeFromKvpValues(it)
             }
 
-            //check if rawScientificName has changed
+            //check if name information has changed
             if (params.rawScientificName.trim() != sli.rawScientificName.trim()) {
                 log.debug "rawScientificName is different: " + params.rawScientificName + " VS " + sli.rawScientificName
                 sli.rawScientificName = params.rawScientificName
+                changed = true
                 // lookup guid
-                helperService.matchNameToSpeciesListItem(sli.rawScientificName, sli)
+                helperService.matchNameToSpeciesListItem(sli.rawScientificName, sli, sli.mylist)
                 //sli.guid = helperService.findAcceptedLsidByScientificName(sli.rawScientificName)?: helperService.findAcceptedLsidByCommonName(sli.rawScientificName)
             }
+            if (changed) {
+                log.debug "re-matching name for ${params.rawScientificName}"
+                helperService.matchNameToSpeciesListItem(sli.rawScientificName, sli, sli.mylist)
+            }
 
             if (!sli.validate()) {
                 def message = "Could not update SpeciesListItem: ${sli.rawScientificName} - " + sli.errors.allErrors

grails-app/controllers/au/org/ala/specieslist/SpeciesListController.groovy

@@ -15,6 +15,7 @@
 package au.org.ala.specieslist
 
 import au.org.ala.web.AuthService
+import au.org.ala.names.ws.api.SearchStyle
 import com.opencsv.CSVReader
 import grails.converters.JSON
 import grails.gorm.transactions.Transactional
@@ -30,6 +31,7 @@ class SpeciesListController {
     private static final String[] ACCEPTED_CONTENT_TYPES = ["text/plain", "text/csv"]
 
     HelperService helperService
+    ColumnMatchingService columnMatchingService
     AuthService authService
     BieService bieService
     BiocacheService biocacheService
@@ -164,6 +166,8 @@ class SpeciesListController {
                             formParams.category,
                             formParams.generalisation,
                             formParams.sdsType,
+                            formParams.looseSearch== null || formParams.looseSearch.isEmpty() ? null : Boolean.parseBoolean(formParams.looseSearch),
+                            formParams.searchStyle == null || formParams.searchStyle.isEmpty() ? null : SearchStyle.valueOf(formParams.searchStyle),
                             header.split(","),
                             vocabs)
 
@@ -472,6 +476,7 @@ class SpeciesListController {
         while (offset < totalRows) {
             List items
             List guidBatch = [], sliBatch = []
+            Map<SpeciesList, List<SpeciesListItem>> batches = new HashMap<>()
             List<SpeciesListItem> searchBatch = new ArrayList<SpeciesListItem>()
             if (id) {
                 items = SpeciesListItem.findAllByDataResourceUid(id, [max: BATCH_SIZE, offset: offset])
@@ -481,23 +486,30 @@ class SpeciesListController {
 
             SpeciesListItem.withSession { session ->
                 items.eachWithIndex { SpeciesListItem item, Integer i ->
+                    SpeciesList speciesList = item.mylist
+                    List<SpeciesListItem> batch = batches.get(speciesList)
+                    if (batch == null) {
+                        batch = new ArrayList<>();
+                        batches.put(speciesList, batch)
+                    }
                     String rawName = item.rawScientificName
-                    log.debug i + ". Rematching: " + rawName
+                    log.debug i + ". Rematching: " + rawName + "/" + speciesList.dataResourceUid
                     if (rawName && rawName.length() > 0) {
-                        searchBatch.add(item)
+                        batch.add(item)
                     } else {
                         item.guid = null
                         if (!item.save(flush: true)) {
                             log.error "Error saving item (" + rawName + "): " + item.errors()
                         }
                     }
                 }
-
-                helperService.matchAll(searchBatch)
-                searchBatch.each {SpeciesListItem item ->
-                    if (item.guid) {
-                        guidBatch.push(item.guid)
-                        sliBatch.push(item)
+                batches.each { list, batch ->
+                    helperService.matchAll(batch, list)
+                    batch.each {SpeciesListItem item ->
+                        if (item.guid) {
+                            guidBatch.push(item.guid)
+                            sliBatch.push(item)
+                        }
                     }
                 }
 
@@ -522,7 +534,7 @@ class SpeciesListController {
     private parseDataFromCSV(CSVReader csvReader, String separator) {
         def rawHeader = csvReader.readNext()
         log.debug(rawHeader.toList()?.toString())
-        def parsedHeader = helperService.parseHeader(rawHeader) ?: helperService.parseData(rawHeader)
+        def parsedHeader = columnMatchingService.parseHeader(rawHeader) ?: helperService.parseData(rawHeader)
         def processedHeader = parsedHeader.header
         log.debug(processedHeader?.toString())
         def dataRows = new ArrayList<String[]>()
@@ -531,7 +543,7 @@ class SpeciesListController {
             dataRows.add(helperService.parseRow(currentLine.toList()))
             currentLine = csvReader.readNext()
         }
-        def nameColumns = helperService.speciesNameColumns + helperService.commonNameColumns
+        def nameColumns = columnMatchingService.speciesNameMatcher.names + columnMatchingService.commonNameMatcher.names
         if (processedHeader.find {
             it == "scientific name" || it == "vernacular name" || it == "common name" || it == "ambiguous name"
         } && processedHeader.size() > 0) {

grails-app/controllers/au/org/ala/specieslist/WebServiceController.groovy

@@ -282,12 +282,16 @@ class WebServiceController {
                 dataResourceUid: sl.dataResourceUid,
                 listName       : sl.listName,
                 dateCreated    : sl.dateCreated,
+                lastUpdated    : sl.lastUpdated,
+                lastUploaded   : sl.lastUploaded,
                 username       : sl.username,
                 fullName       : sl.getFullName(),
                 itemCount      : sl.itemsCount,//SpeciesListItem.countByList(sl)
                 isAuthoritative: (sl.isAuthoritative ?: false),
                 isInvasive     : (sl.isInvasive ?: false),
-                isThreatened   : (sl.isThreatened ?: false)
+                isThreatened   : (sl.isThreatened ?: false),
+                looseSearch    : sl.looseSearch,
+                searchStyle    : sl.searchStyle?.toString()
             ]
             if (sl.listType) {
                 retValue["listType"] = sl?.listType?.toString()
@@ -313,11 +317,13 @@ class WebServiceController {
             def listCounts = allLists.totalCount
             def retValue = [listCount: listCounts, sort: params.sort, order: params.order, max: params.max, offset: params.offset,
                             lists    : allLists.collect {
-                                [dataResourceUid: it.dataResourceUid,
+                                [
+                                 dataResourceUid: it.dataResourceUid,
                                  listName       : it.listName,
                                  listType       : it?.listType?.toString(),
                                  dateCreated    : it.dateCreated,
                                  lastUpdated    : it.lastUpdated,
+                                 lastUploaded   : it.lastUploaded,
                                  username       : it.username,
                                  fullName       : it.getFullName(),
                                  itemCount      : it.itemsCount,
@@ -328,9 +334,12 @@ class WebServiceController {
                                  sdsType        : it.sdsType,
                                  isAuthoritative: it.isAuthoritative ?: false,
                                  isInvasive     : it.isInvasive ?: false,
-                                 isThreatened   : it.isThreatened ?: false]
-                            }]
+                                 isThreatened   : it.isThreatened ?: false,
+                                 looseSearch    : it.looseSearch,
+                                 searchStyle    : it.searchStyle?.toString()
+                                ]
 
+                            }]
             render retValue as JSON
         }
     }

grails-app/domain/au/org/ala/specieslist/SpeciesList.groovy

@@ -15,6 +15,8 @@
 
 package au.org.ala.specieslist
 
+import au.org.ala.names.ws.api.SearchStyle
+
 class SpeciesList {
     def authService
 
@@ -29,6 +31,7 @@ class SpeciesList {
     String wkt
     Date dateCreated
     Date lastUpdated
+    Date lastUploaded
     ListType listType
     Boolean isPrivate
     Boolean isSDS
@@ -42,8 +45,9 @@ class SpeciesList {
     String generalisation
     String category
     String sdsType
+    Boolean looseSearch // if undefined use the server default
+    SearchStyle searchStyle // if undefined use the server default
     String ownerFullName // derived by concatenating the firstName and surname fields
-
     static transients = [ "fullName" ]
 
     static hasMany = [items: SpeciesListItem, editors: String]
@@ -67,6 +71,9 @@ class SpeciesList {
         generalisation(nullable: true)
         authority(nullable:  true)
         sdsType nullable:  true
+        looseSearch nullable: true
+        searchStyle nullable: true
+        lastUploaded nullable: true
         userId nullable: true
         ownerFullName nullable: true // derived
     }

grails-app/i18n/messages.properties

@@ -95,6 +95,9 @@ speciesList.authority.label=SDS Authority
 speciesList.category.label=SDS Category
 speciesList.generalisation.label=SDS Coordinate Generalisation
 speciesList.sdsType.label=SDS Type
+speciesList.looseSearch.label=Loose Name Search
+speciesList.searchStyle.label=Name Search Style
+speciesList.lastUploaded.label=Date last uploaded
 
 upload.heading.hasList=Resubmit to an existing list
 upload.heading=Upload a list
@@ -371,7 +374,9 @@ public.lists.items.header01=Supplied Name
 public.lists.items.header02=Scientific Name (matched)
 public.lists.items.header03=Author (matched)
 public.lists.items.header04=Common Name (matched)
-public.lists.items.header05=Image
+public.lists.items.header05=Family (matched)
+public.lists.items.header06=Kingdom (matched)
+public.lists.items.header07=Image
 generic.lists.label.reload= - Page will be reloaded ...
 public.lists.items.edit.error=Required field: supplied name cannot be blank