#329 Fix spatial layer query settings

Improve admin page and fix bugs on evaluate methods refactor styles
AtlasOfLivingAustralia · Dec 2, 2024 · f05b216 · f05b216
1 parent 95a4b02
commit f05b216
Show file tree

Hide file tree

Showing 14 changed files with 124 additions and 78 deletions.
diff --git a/README.md b/README.md
@@ -44,15 +44,15 @@ Change logs
 DB schema update:
 ``` ALTER TABLE alerts.query_result MODIFY logs TEXT NULL; ```
 
-### 4.2.0 Release
+### 4.3.0 Release
 No DB changes
 
 
-### 4.3.1 Release
+### 4.4.0 Release
 Apply new templates for alerts
 
 #### Change logs
-Database update: Check RELEASE.MD for the queries to run
+Database update: Check release/4.4.0-release.sql
 
 
 
diff --git a/grails-app/controllers/au/org/ala/alerts/AdminController.groovy b/grails-app/controllers/au/org/ala/alerts/AdminController.groovy
@@ -367,7 +367,7 @@ class AdminController {
 
         def frequency = 'weekly'
         QueryResult qr = notificationService.getQueryResult(query, Frequency.findByName(frequency))
-        qr.lastResult = notificationService.gzipResult(processedJson)
+        qr.lastResult = qr.compress(processedJson)
         //this logic only applies on preview page
         qr.previousCheck = qr.lastChecked
         qr.lastChecked = since
@@ -472,7 +472,7 @@ class AdminController {
 
             def frequency = 'weekly'
             QueryResult qr = notificationService.getQueryResult(query, Frequency.findByName(frequency))
-            qr.lastResult = notificationService.gzipResult(processedJson)
+            qr.lastResult = qr.compress(processedJson)
             File tempCSV = biosecurityCSVService.createTempCSV(qr)
             csvFiles.add(tempCSV.path)
         }

diff --git a/grails-app/controllers/au/org/ala/alerts/NotificationController.groovy b/grails-app/controllers/au/org/ala/alerts/NotificationController.groovy
@@ -10,6 +10,7 @@ class NotificationController {
     def userService
     def authService
     def diffService
+    def queryResultService
 
     static allowedMethods = [save: "POST", update: "POST", delete: "POST"]
 
@@ -124,16 +125,22 @@ class NotificationController {
 
     def evaluateChangeDetectionAlgorithm = {
         def query = Query.get(params.queryId)
-        def queryResult = query?.queryResults?.find { queryResult ->
-            queryResult.id == params.queryResultId.toLong()
-        }
+
+        def queryResult = queryResultService.get(params.queryResultId)
         //NOTE: this is a hack since the lastResult will be copied into the previousResult in RefreshProperties method
         //We need to hack the current result with previousResult.
         def lastResult = queryResult.decompress(queryResult.lastResult)
         queryResult.lastResult = queryResult.previousResult
+        //Assume ONLY one property value
+        queryResult.propertyValues?[0]?.currentValue = queryResult.propertyValues?[0]?.previousValue
 
         notificationService.refreshProperties(queryResult, lastResult)
         boolean hasChanged = diffService.hasChanged(queryResult)
+
+        //Update the results
+        queryResult.previousResult = queryResult.lastResult
+        queryResult.lastResult =  queryResult.compress(lastResult)
+
         def records = notificationService.collectUpdatedRecords(queryResult)
         def results = ["hasChanged": hasChanged, "brief": queryResult.brief(),  "records": records]
         render results as JSON

diff --git a/grails-app/domain/au/org/ala/alerts/QueryResult.groovy b/grails-app/domain/au/org/ala/alerts/QueryResult.groovy
@@ -130,11 +130,15 @@ class QueryResult {
 
     byte[] compress(String json) {
         //store the last result from the webservice call
-        ByteArrayOutputStream bout = new ByteArrayOutputStream()
-        GZIPOutputStream gzout = new GZIPOutputStream(bout)
-        gzout.write(json.toString().getBytes())
-        gzout.flush()
-        gzout.finish()
-        bout.toByteArray()
+        if (json) {
+            ByteArrayOutputStream bout = new ByteArrayOutputStream()
+            GZIPOutputStream gzout = new GZIPOutputStream(bout)
+            gzout.write(json.toString().getBytes())
+            gzout.flush()
+            gzout.finish()
+            bout.toByteArray()
+        } else {
+            null
+        }
     }
 }
diff --git a/grails-app/services/au/org/ala/alerts/DiffService.groovy b/grails-app/services/au/org/ala/alerts/DiffService.groovy
@@ -186,43 +186,45 @@ class DiffService {
     def getNewRecordsFromDiff(QueryResult queryResult) {
 
         def records = []
-
+        String last = "{}"
+        String previous = "{}"
         if (queryResult.lastResult != null ) {
-            String last = decompressZipped(queryResult.lastResult)
-            String previous = "{}"
-            // If previous result is null, assign an empty Json object
-            if ( queryResult.previousResult != null) {
-                previous = decompressZipped(queryResult.previousResult)
-            }
+            last = decompressZipped(queryResult.lastResult)
+        }
 
-            try {
-                if (!last.startsWith("<") && !previous.startsWith("<")) {
-                    // Don't try and process 401, 301, 500, etc., responses that contain HTML
-                    if (queryService.isMyAnnotation(queryResult.query)) {
-                        // for normal alerts, comparing occurrence uuid is enough to show the difference.
-                        // for my annotation alerts, same occurrence record could exist in both result but have different assertions.
-                        // so comparing occurrence uuid is not enough, we need to compare 50001/50002/50003 sections inside each occurrence record
-                        records = myAnnotationService.diff(previous, last, queryResult.query.recordJsonPath)
-                    } else if (queryService.isAnnotation(queryResult.query)) {
-                        records = annotationService.diff(previous, last, queryResult.query.recordJsonPath)
-                    } else if (queryService.isDatasetQuery(queryResult.query)) {
-                        records = datasetService.diff(queryResult)
-                    } else if (queryService.isDatasetResource(queryResult.query)) {
-                        records = dataResourceService.diff(queryResult)
-                    } else if ( queryService.isBiocacheImages(queryResult.query)) {
-                        records = imageService.diff(queryResult)
-                    } else if ( queryService.isBiocacheImages(queryResult.query)) {
-                        records = datasetService.diff(queryResult)
-                    } else {
-                        records = findNewRecordsById(previous, last, queryResult.query.recordJsonPath, queryResult.query.idJsonPath)
-                    }
+        // If previous result is null, assign an empty Json object String
+        if ( queryResult.previousResult != null) {
+            previous = decompressZipped(queryResult.previousResult)
+        }
+
+        try {
+            if (!last.startsWith("<") && !previous.startsWith("<")) {
+                // Don't try and process 401, 301, 500, etc., responses that contain HTML
+                if (queryService.isMyAnnotation(queryResult.query)) {
+                    // for normal alerts, comparing occurrence uuid is enough to show the difference.
+                    // for my annotation alerts, same occurrence record could exist in both result but have different assertions.
+                    // so comparing occurrence uuid is not enough, we need to compare 50001/50002/50003 sections inside each occurrence record
+                    records = myAnnotationService.diff(previous, last, queryResult.query.recordJsonPath)
+                } else if (queryService.isAnnotation(queryResult.query)) {
+                    records = annotationService.diff(previous, last, queryResult.query.recordJsonPath)
+                } else if (queryService.isDatasetQuery(queryResult.query)) {
+                    records = datasetService.diff(queryResult)
+                } else if (queryService.isDatasetResource(queryResult.query)) {
+                    records = dataResourceService.diff(queryResult)
+                } else if ( queryService.isBiocacheImages(queryResult.query)) {
+                    records = imageService.diff(queryResult)
+                } else if ( queryService.isBiocacheImages(queryResult.query)) {
+                    records = datasetService.diff(queryResult)
                 } else {
-                    log.warn "queryId: " + queryResult.query.id + ", queryResult:" + queryResult.id + " last or previous objects contains HTML and not JSON"
+                    records = findNewRecordsById(previous, last, queryResult.query.recordJsonPath, queryResult.query.idJsonPath)
                 }
-            } catch (Exception ex) {
-                log.error("queryId: ${queryResult.query.id}, JsonPath error: ${ex}")
+            } else {
+                log.warn "queryId: " + queryResult.query.id + ", queryResult:" + queryResult.id + " last or previous objects contains HTML and not JSON"
             }
+        } catch (Exception ex) {
+            log.error("queryId: ${queryResult.query.id}, JsonPath error: ${ex}")
         }
+
         return records
     }
 

diff --git a/grails-app/services/au/org/ala/alerts/NotificationService.groovy b/grails-app/services/au/org/ala/alerts/NotificationService.groovy
@@ -10,7 +10,6 @@ import org.grails.web.json.JSONElement
 import org.grails.web.json.JSONObject
 
 import javax.transaction.Transactional
-import java.util.zip.GZIPOutputStream
 import java.text.SimpleDateFormat
 import groovy.time.TimeCategory
 import org.hibernate.FlushMode
@@ -143,7 +142,7 @@ class NotificationService {
             qr.previousCheck = qr.lastChecked
             // store the last result from the webservice call
             qr.previousResult = qr.lastResult
-            qr.lastResult = gzipResult(processedJson)
+            qr.lastResult = qr.compress(processedJson)
             qr.lastChecked = checkDate
             //todo: review this algorithm for all queries
             qr.hasChanged = diffService.hasChanged(qr)
@@ -287,15 +286,7 @@ class NotificationService {
         qcr
     }
 
-    byte[] gzipResult(String json) {
-        //store the last result from the webservice call
-        ByteArrayOutputStream bout = new ByteArrayOutputStream()
-        GZIPOutputStream gzout = new GZIPOutputStream(bout)
-        gzout.write(json.toString().getBytes())
-        gzout.flush()
-        gzout.finish()
-        bout.toByteArray()
-    }
+
 
 
     /**

diff --git a/grails-app/services/au/org/ala/alerts/QueryResultService.groovy b/grails-app/services/au/org/ala/alerts/QueryResultService.groovy
@@ -18,6 +18,7 @@ package au.org.ala.alerts
  * Database service for QueryResult:
  */
 class QueryResultService {
+    def queryService
     /**
      * Get QueryResult by id, including cascading objects: query and frequency
      *
@@ -27,11 +28,14 @@ class QueryResultService {
     def get(id){
         QueryResult qs = QueryResult.get(id)
         if (qs) {
-            Query query = Query.get(qs.query.id)
+            Query query = queryService.get(qs.query.id)
             qs.query = query
 
             Frequency frequency = Frequency.get(qs.frequency.id)
             qs.frequency = frequency
+
+            PropertyValue pvs = PropertyValue.findByQueryResult(qs)
+            qs.propertyValues = [pvs]
         }
         return qs
     }

diff --git a/grails-app/services/au/org/ala/alerts/QueryService.groovy b/grails-app/services/au/org/ala/alerts/QueryService.groovy
@@ -15,6 +15,10 @@ class QueryService {
     def messageSource, dataSource
     def siteLocale = new Locale.Builder().setLanguageTag(Holders.config.siteDefaultLanguage as String).build()
 
+    def get(id){
+        Query.get(id)
+    }
+
     Notification getNotificationForUser(Query query, User user) {
         Notification n = null
         //find the query

diff --git a/grails-app/views/admin/query.gsp b/grails-app/views/admin/query.gsp
@@ -75,10 +75,7 @@
                                        <g:if test="${query.queryResults?.size() > 0}">
                                             <ul>
                                                 <g:each var="queryResult" in="${query.queryResults.sort { it.frequency?.name }}">
-                                                    <div>
-                                                        [<g:link controller="queryResult" action="getDetails" params="[id: queryResult.id]" target="_blank">${queryResult.id}</g:link>] <b title="Query Result ID:${queryResult.id}"> ${queryResult.frequency?.name} subscribers: </b> <span class="badge badge-primary">${query.countSubscribers(queryResult.frequency?.name)}</span>
-                                                        - Last checked: <g:link controller="ws" action="getQueryLogs" params="[id: query.id, frequency: queryResult.frequency?.name]" target="_blank"> <i class="fa fa-info-circle" aria-hidden="true"></i>${queryResult?.lastChecked}</g:link>
-                                                    </div>
+
                                                     <div>
                                                         <g:if test="${queryResult.hasChanged}">
                                                             <span class="badge badge-info">Changed</span>
@@ -87,12 +84,26 @@
                                                             <span class="badge badge-dark">No changes</span>
                                                         </g:else>
                                                     </div>
+                                                    <div>
+                                                        <b>${queryResult.frequency?.name?.toUpperCase()}</b> query result ID: <g:link controller="queryResult" action="getDetails" params="[id: queryResult.id]" target="_blank"> <span class="badge badge-primary">${queryResult.id}</span></g:link>
+                                                        <br/>
+                                                        <g:if test="${queryResult?.lastChecked}">
+                                                             Last checked: ${queryResult?.lastChecked}&nbsp;&nbsp;
+                                                        </g:if>
+                                                        <g:link controller="ws" action="getQueryLogs" params="[id: query.id, frequency: queryResult.frequency?.name]" target="_blank">Log</g:link>
+                                                        &nbsp;&nbsp;
+                                                        <g:if test="${queryResult?.queryUrlUsed}">
+                                                            <a href="${queryResult?.queryUrlUsed}" target="_blank" title="URL for search">
+                                                                Query URL
+                                                            </a>
+                                                        </g:if>
+                                                        <br/>
+                                                        Subscribers:${query.countSubscribers(queryResult.frequency?.name)}
+                                                    </div>
                                                     <div>
                                                         <g:each var="pv" in="${queryResult.propertyValues}">
-                                                            ${pv.propertyPath}<br>
-                                                            Current Value: ${pv.currentValue}  &nbsp;
-                                                            Previous Value: ${pv.previousValue} <br>
-
+                                                            <span class="badge badge-light">${pv.propertyPath.id}</span> ${pv.propertyPath}<br>
+                                                            <span class="badge badge-light">${pv.id}</span> Current Value: ${pv.currentValue}; Previous Value: ${pv.previousValue} <br>
                                                         </g:each>
                                                     </div>
                                                     <div style="text-align: right;">
@@ -104,7 +115,7 @@
                                                          </div>
                                                         <g:if test="${queryType != 'biosecurity'}">
                                                             <div style="padding: 5px;">
-                                                            <label>Get the latest records, compare with the current records in the database, Email me the results </label><g:link class="btn btn-info"  controller="admin" action="emailMeLastCheck" params="[queryId: query.id, frequency: queryResult.frequency?.name]" target="_blank">
+                                                            <label>Get the latest records, compare with the current result in the database, Email me the results </label><g:link class="btn btn-info"  controller="admin" action="emailMeLastCheck" params="[queryId: query.id, frequency: queryResult.frequency?.name]" target="_blank">
                                                                 Email me
                                                             </g:link>
                                                             </div>
@@ -121,7 +132,20 @@
                                                                     %>
                                                                     <input type="hidden" name="queryId" value="${query.id}" />
                                                                     <input type="hidden" name="frequency" value="${queryResult.frequency?.name}" />
-                                                                    <label for="checkDate">Collect records on the given date, and email new records </label>
+                                                                    <label for="checkDate">Run the query against the given date, and email new records
+                                                                        <i class="fa fa-info-circle" aria-hidden="true" style="color: #c44d34;"
+                                                                           title="It may be set as starting from that date, ending on that date, spanning a period around that date, or not used at all."></i>
+                                                                        The date range associated with the given date is determined by the
+
+                                                                        <g:if test="${queryResult?.queryUrlUsed}">
+                                                                            <a href="${queryResult?.queryUrlUsed}" target="_blank" title="URL for search">
+                                                                                <i class="fa fa-search" aria-hidden="true"></i> query
+                                                                            </a>
+                                                                        </g:if>
+                                                                        <g:else>
+                                                                             query
+                                                                        </g:else>
+                                                                     </label>
                                                                      <input type="date" id="checkDate" name="checkDate"value="${today}" class="form-control" />
                                                                     <button type="submit" class="btn btn-info mb-2">Email me, No DB update</button>
                                                                 </g:form>

diff --git a/grails-app/views/email/annotations.gsp b/grails-app/views/email/annotations.gsp
@@ -69,7 +69,7 @@
                                         %>
                                         <b>Comment:</b><br/>
                                         <i>${StringUtils.abbreviate(latestAssertion.comment, 100)}</i>
-                                        <br>- <b>${latestAssertion.userDisplayName}
+                                        <br>-<b>${latestAssertion.userDisplayName}
                                             <g:if test="${latestAssertion.created}">,&nbsp;
                                                     <%
                                                         try {

diff --git a/grails-app/views/email/datasets.gsp b/grails-app/views/email/datasets.gsp
@@ -47,8 +47,9 @@ a {
                             <table style="width: 100%">
                                 <tr style="vertical-align: top;">
                                     <td style="width: 70%">
+                                        <strong>${i+1}.</strong>
                                         <a href="${oclink}" style="color: #003A70;font-family: 'Arial', sans-serif;font-size: 16px;line-height: 1.5;">
-                                                <strong>${i+1}. <em>${oc.name}</em></strong>
+                                                <strong><em>${oc.name}</em></strong>
                                         </a>
                                         <p style="padding-left: 15px;">