diff --git a/grails-app/conf/application.yml b/grails-app/conf/application.yml index b1bac43..76b5a6e 100644 --- a/grails-app/conf/application.yml +++ b/grails-app/conf/application.yml @@ -132,7 +132,7 @@ skin: orgSupportEmail: support@ala.org.au privacyPolicy: "https://www.ala.org.au/terms-of-use/privacy-policy/" siteDefaultLanguage: "en" - +customUserAgent: 'alerts/@info.app.version@' security: cas: adminRole: ROLE_ADMIN @@ -186,7 +186,7 @@ biosecurity: bucket: alerts # read lga from the layer lga: LGA2023 - + subscriptionsPerPage: 100 legacy: aus: cl927:* act: cl927:"Australian Capital Territory" OR cl927:"Jervis Bay Territory" diff --git a/grails-app/services/au/org/ala/alerts/AlertsWebService.groovy b/grails-app/services/au/org/ala/alerts/AlertsWebService.groovy index ff0c0fb..b999aee 100644 --- a/grails-app/services/au/org/ala/alerts/AlertsWebService.groovy +++ b/grails-app/services/au/org/ala/alerts/AlertsWebService.groovy @@ -30,6 +30,8 @@ class AlertsWebService { try { conn.setConnectTimeout(10000) conn.setReadTimeout(50000) + conn.setRequestProperty('User-Agent', grailsApplication.config.getProperty("customUserAgent", "ALA-alerts")) + if (apiKey != null) { conn.setRequestProperty('apiKey', apiKey) } diff --git a/grails-app/services/au/org/ala/alerts/BiosecurityService.groovy b/grails-app/services/au/org/ala/alerts/BiosecurityService.groovy index 61d5989..5e691b3 100644 --- a/grails-app/services/au/org/ala/alerts/BiosecurityService.groovy +++ b/grails-app/services/au/org/ala/alerts/BiosecurityService.groovy @@ -163,7 +163,8 @@ class BiosecurityService { while (repeat) { def url = grailsApplication.config.getProperty('lists.baseURL') + "/ws/speciesListItemsInternal/" + drId + "?includeKVP=true" + "&offset=" + offset + "&max=" + max - def speciesList = webService.get(url, [:], ContentType.APPLICATION_JSON, true, false) + def headers = ["User-Agent": "${grailsApplication.config.getProperty("customUserAgent", "alerts")}"] + def speciesList = webService.get(url, [:], ContentType.APPLICATION_JSON, true, false, headers) if (speciesList.statusCode != 200 && speciesList.statusCode != 201) { log.error("Failed to access: " + url) log.error("Error: " + speciesList.error) @@ -237,7 +238,10 @@ class BiosecurityService { log.debug("URL: " + url) try { - def get = JSON.parse(new URL(url).text) + def get = JSON.parse(new URL(url).openConnection().with { conn -> + conn.setRequestProperty("User-Agent", grailsApplication.config.getProperty("customUserAgent", "alerts")) + conn.inputStream.text + }) get?.occurrences?.each { occurrence -> occurrences[occurrence.uuid] = occurrence //extra info should be added here @@ -261,7 +265,10 @@ class BiosecurityService { def fetchExtraInfo(def uuid, def occurrence) { try { def url = grailsApplication.config.getProperty('biocacheService.baseURL') + '/occurrences/' + uuid - def record = JSON.parse(new URL(url).text) + def record = JSON.parse(new URL(url).openConnection().with { conn -> + conn.setRequestProperty("User-Agent", grailsApplication.config.getProperty("customUserAgent", "alerts")) + conn.inputStream.text + }) occurrence['firstLoaded'] = record.raw?.firstLoaded //Do not join, let CSV generate handle it occurrence['cl'] = record.processed?.cl?.collect { "${it.key}:${it.value}" } diff --git a/grails-app/services/au/org/ala/alerts/NotificationService.groovy b/grails-app/services/au/org/ala/alerts/NotificationService.groovy index c3e07b6..3df5ac8 100644 --- a/grails-app/services/au/org/ala/alerts/NotificationService.groovy +++ b/grails-app/services/au/org/ala/alerts/NotificationService.groovy @@ -81,7 +81,7 @@ class NotificationService { if (!urlString.contains("___MAX___")) { // queries without paging if (!queryService.isBioSecurityQuery(query)) { - processedJson = processQueryReturnedJson(query, IOUtils.toString(new URL(urlString).newReader())) + processedJson = processQueryReturnedJson(query, getWebserviceResults(urlString)) } } else { // queries with paging @@ -90,7 +90,24 @@ class NotificationService { def result = [] boolean finished = false def allLists = [] - while (!finished && (result = processQueryReturnedJson(query, new URL(urlString.replaceAll('___MAX___', String.valueOf(max)).replaceAll('___OFFSET___', String.valueOf(offset))).text))?.size()) { + + while (!finished) { + // Construct the URL + def url = new URL(urlString + .replaceAll('___MAX___', String.valueOf(max)) + .replaceAll('___OFFSET___', String.valueOf(offset)) + ) + + // Process the query + result = processQueryReturnedJson(query, getWebserviceResults(url.toString())) // API errors will result in an empty string ("") + + // Check if we have results + if (!result || result?.size() == 0) { + finished = true + continue + } + + // Increment offset for next iteration offset += max try { @@ -192,7 +209,7 @@ class NotificationService { // queries without paging if (!queryService.isBioSecurityQuery(query)) { // standard query - processedJson = processQueryReturnedJson(query, IOUtils.toString(new URL(urlString).newReader())) + processedJson = processQueryReturnedJson(query, getWebserviceResults(urlString)) } else { // biosecurity query is handled elsewhere Date since = lastQueryResult.lastChecked ?: DateUtils.addDays(new Date(), -1 * grailsApplication.config.getProperty("biosecurity.legacy.firstLoadedDateAge", Integer, 7)) @@ -203,25 +220,39 @@ class NotificationService { // queries with paging int max = PAGING_MAX int offset = 0 - def result - boolean finished = false def allLists = [] - while (!finished && (result = processQueryReturnedJson(query, new URL(urlString.replaceAll('___MAX___', String.valueOf(max)).replaceAll('___OFFSET___', String.valueOf(offset))).text))?.size()) { - offset += max + boolean finished = false + + while (!finished) { + // Construct the URL with max and offset values + String urlWithParams = urlString.replace('___MAX___', max.toString()).replace('___OFFSET___', offset.toString()) + + // Get the result from the query + def result = processQueryReturnedJson(query, getWebserviceResults(urlWithParams)) + + // Check if result is not empty + if (result?.size() == 0) { + finished = true + break + } try { + // Read latest values from JSON def latestValue = JsonPath.read(result, query.recordJsonPath) + if (latestValue.size() == 0) { finished = true } else { processedJson = result allLists.addAll(latestValue) + offset += max // Update offset for the next iteration } } catch (Exception e) { - //expected behaviour for missing properties + // Handle missing properties gracefully finished = true } } + // only for species lists def json = JSON.parse(processedJson) as JSONObject if (json.lists) { @@ -443,7 +474,13 @@ class NotificationService { return json } - JSONObject rslt = JSON.parse(json) as JSONObject + JSONObject rslt + try { + rslt = JSON.parse(json) as JSONObject + } catch (Exception e) { + log.error("Failed to parse JSON in processQueryReturnedJson(): ${e.message}", e) + rslt = new JSONObject() + } // all the occurrences user has made annotations to if (rslt.occurrences) { @@ -477,17 +514,30 @@ class NotificationService { return rslt.toString() } - JSONElement getJsonElements(String url) { + String getWebserviceResults(String url) { log.debug "(internal) getJson URL = " + url def conn = new URL(url).openConnection() try { conn.setConnectTimeout(10000) conn.setReadTimeout(50000) - return JSON.parse(conn.getInputStream(), "UTF-8") + conn.setRequestProperty('User-Agent', grailsApplication.config.getProperty("customUserAgent", "ALA-alerts")) + return IOUtils.toString(conn.getInputStream(), "UTF-8") } catch (Exception e) { def error = "Failed to get json from web service (${url}). ${e.getClass()} ${e.getMessage()}, ${e}" log.error error - return new JSONObject(); + return "" + } + } + + JSONElement getJsonElements(String url) { + log.debug "(internal) getJson URL = " + url + def data = getWebserviceResults(url) + + try { + return JSON.parse(data) + } catch (Exception e) { + log.error("Failed to parse JSON in getJsonElements() (value: ${data}): ${e.message}", e) + return new JSONObject() } }