diff --git a/.travis.yml b/.travis.yml index 09eb38fa..aeea0607 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,14 +1,10 @@ language: groovy jdk: - openjdk11 -sudo: false branches: only: - master - develop - - /^feature\/.*$/ - - /^bugfix\/.*$/ - before_cache: - rm -f $HOME/.gradle/caches/modules-2/modules-2.lock - rm -fr $HOME/.gradle/caches/*/plugin-resolution/ @@ -19,6 +15,10 @@ cache: - $HOME/.gradle/caches/ - $HOME/.gradle/wrapper/ +install: + - 'travis_wait 30 ./gradlew clean' + - './gradlew assemble' + after_success: - './gradlew bootJar' - '[ "${TRAVIS_PULL_REQUEST}" = "false" ] && travis_retry ./gradlew publish' @@ -27,4 +27,4 @@ env: global: - JAVA_TOOL_OPTIONS=-Dhttps.protocols=TLSv1.2 - secure: G31IscTcNtjjFA0R2pDRaZ2Std1d5F3pIfr3NUe0PsKcj8F2ubFd6xZ3LCccKu/HdDdlJ/6K6Khn1EfwHie5NRP+Uu7oAUOVkTXSeI4KfQFywpf3TvKLILh8/2NUe0451ESivKQ28UW4jPOehmLtCyd7MbxnO9GIrYyaWjNEJJw= - - secure: g7txXzs6g/DzF9bwI5UZ4wG2QxB4aqQeAK1TxL3P0DfMHB8hXs/BydKrFaA9TnJiB/g0fwabNCuJWr+IgOL3EPA1txwVu9VVttHqKiJXthoen1eN0QJtsCWhlE2/ldC1mENaJVXBQs7++sNutytUIOr9m2Yk78mXnu2RIfk7U/k= \ No newline at end of file + - secure: g7txXzs6g/DzF9bwI5UZ4wG2QxB4aqQeAK1TxL3P0DfMHB8hXs/BydKrFaA9TnJiB/g0fwabNCuJWr+IgOL3EPA1txwVu9VVttHqKiJXthoen1eN0QJtsCWhlE2/ldC1mENaJVXBQs7++sNutytUIOr9m2Yk78mXnu2RIfk7U/k= diff --git a/build.gradle b/build.gradle index 034d9532..8794a08c 100644 --- a/build.gradle +++ b/build.gradle @@ -1,34 +1,23 @@ buildscript { - repositories { - mavenLocal() - maven { url "https://nexus.ala.org.au/content/groups/public/" } - maven { url "https://repo.grails.org/grails/core" } - } - dependencies { - classpath "org.grails:grails-gradle-plugin:$grailsVersion" - classpath "org.grails.plugins:hibernate5:${gormVersion}" - classpath "gradle.plugin.com.github.erdi.webdriver-binaries:webdriver-binaries-gradle-plugin:$webdriverBinariesVersion" - classpath "com.bertramlabs.plugins:asset-pipeline-gradle:3.4.6" - classpath 'org.grails.plugins:database-migration:4.2.0' - } + version "5.0.0-SNAPSHOT" + group "au.org.ala" } plugins { - id "com.gorylenko.gradle-git-properties" version "2.4.1" -} - -version "4.0.0" -group "au.org.ala" + id "groovy" + id "org.grails.grails-gsp" + id "org.grails.grails-web" + id "com.github.erdi.webdriver-binaries" version "3.0" + id "war" + id "idea" + id "com.bertramlabs.asset-pipeline" + id "application" + id "eclipse" -apply plugin:"eclipse" -apply plugin:"idea" -apply plugin:"war" -apply plugin:"org.grails.grails-web" -apply plugin:"com.github.erdi.webdriver-binaries" -apply plugin:"com.bertramlabs.asset-pipeline" -apply plugin:"org.grails.grails-gsp" -apply plugin:"maven-publish" + id "com.gorylenko.gradle-git-properties" version "2.4.1" + id "maven-publish" +} publishing { targetCompatibility = 1.11 @@ -51,12 +40,22 @@ publishing { bootWar { launchScript() + dependsOn(compileGroovyPages) +} + +war { + dependsOn(compileGroovyPages) +} + +java { + sourceCompatibility = JavaVersion.toVersion("11") } repositories { mavenLocal() maven { url "https://nexus.ala.org.au/content/groups/public/" } - maven { url "https://repo.grails.org/grails/core" } + mavenCentral() + maven { url "https://repo.grails.org/grails/core/" } } configurations { @@ -94,7 +93,7 @@ dependencies { implementation "org.grails.plugins:scaffolding" implementation "org.grails.plugins:events" implementation "org.grails.plugins:hibernate5" - implementation "org.hibernate:hibernate-core:5.4.18.Final" + implementation("org.hibernate:hibernate-core:5.6.15.Final") implementation "org.grails.plugins:gsp" implementation "com.opencsv:opencsv:3.7" @@ -106,8 +105,8 @@ dependencies { // See: https://github.com/AtlasOfLivingAustralia/collectory/issues/84#issuecomment-1070670979 // before updating mysql-connector-java - implementation "mysql:mysql-connector-java:8.0.22" - implementation "org.grails.plugins:ala-bootstrap3:4.1.0" + implementation 'mysql:mysql-connector-java:8.0.33' + implementation "org.grails.plugins:ala-bootstrap3:4.4.0" implementation "au.org.ala.plugins.grails:ala-charts-plugin:2.3.0" implementation "org.grails.plugins:ala-auth:$alaSecurityLibsVersion" implementation "org.grails.plugins:ala-ws-security-plugin:$alaSecurityLibsVersion" @@ -123,23 +122,23 @@ dependencies { runtimeOnly "com.h2database:h2" runtimeOnly "org.apache.tomcat:tomcat-jdbc" runtimeOnly "javax.xml.bind:jaxb-api:2.3.1" - runtimeOnly "com.bertramlabs.plugins:asset-pipeline-grails:3.4.6" + runtimeOnly "com.bertramlabs.plugins:asset-pipeline-grails:4.3.0" testImplementation "io.micronaut:micronaut-inject-groovy" testImplementation "org.grails:grails-gorm-testing-support" testImplementation "org.mockito:mockito-core" testImplementation "org.grails:grails-web-testing-support" - testImplementation "org.grails.plugins:geb" - testImplementation "org.seleniumhq.selenium:selenium-remote-driver:$seleniumVersion" - testImplementation "org.seleniumhq.selenium:selenium-api:$seleniumVersion" - testImplementation "org.seleniumhq.selenium:selenium-support:$seleniumVersion" - runtimeOnly "net.sourceforge.htmlunit:htmlunit:2.18" - testImplementation "org.seleniumhq.selenium:selenium-htmlunit-driver:2.47.1" - testImplementation "org.seleniumhq.selenium:selenium-chrome-driver:$seleniumVersion" - testImplementation "org.seleniumhq.selenium:selenium-firefox-driver:$seleniumVersion" - testImplementation "org.seleniumhq.selenium:selenium-safari-driver:$seleniumSafariDriverVersion" +// testImplementation "org.grails.plugins:geb" +// testImplementation "org.seleniumhq.selenium:selenium-remote-driver:$seleniumVersion" +// testImplementation "org.seleniumhq.selenium:selenium-api:$seleniumVersion" +// testImplementation "org.seleniumhq.selenium:selenium-support:$seleniumVersion" +// runtimeOnly "net.sourceforge.htmlunit:htmlunit:2.18" +// testImplementation "org.seleniumhq.selenium:selenium-htmlunit-driver:2.47.1" +// testImplementation "org.seleniumhq.selenium:selenium-chrome-driver:$seleniumVersion" +// testImplementation "org.seleniumhq.selenium:selenium-firefox-driver:$seleniumVersion" +// testImplementation "org.seleniumhq.selenium:selenium-safari-driver:$seleniumSafariDriverVersion" implementation 'org.grails.plugins:sentry:11.7.25' // db-migration - implementation 'org.liquibase:liquibase-core:4.19.0' + implementation 'org.liquibase:liquibase-core:4.20.0' implementation('org.grails.plugins:database-migration:4.2.0') { // spring-boot-cli exclusion required since Grails5 upgrade to prevent NullPointerException Error: https://github.com/grails/grails-database-migration/issues/268 exclude module: 'spring-boot-cli' @@ -157,18 +156,18 @@ bootRun { String springProfilesActive = 'spring.profiles.active' systemProperty springProfilesActive, System.getProperty(springProfilesActive) } - -tasks.withType(Test) { - systemProperty "geb.env", System.getProperty('geb.env') - systemProperty "geb.build.reportsDir", reporting.file("geb/integrationTest") - systemProperty "webdriver.chrome.driver", System.getProperty('webdriver.chrome.driver') - systemProperty "webdriver.gecko.driver", System.getProperty('webdriver.gecko.driver') -} - -webdriverBinaries { - chromedriver "$chromeDriverVersion" - geckodriver "$geckodriverVersion" -} +// +//tasks.withType(Test) { +// systemProperty "geb.env", System.getProperty('geb.env') +// systemProperty "geb.build.reportsDir", reporting.file("geb/integrationTest") +// systemProperty "webdriver.chrome.driver", System.getProperty('webdriver.chrome.driver') +// systemProperty "webdriver.gecko.driver", System.getProperty('webdriver.gecko.driver') +//} +// +//webdriverBinaries { +// chromedriver "$chromeDriverVersion" +// geckodriver "$geckodriverVersion" +//} assets { minifyJs = true diff --git a/gradle.properties b/gradle.properties index cb00d8fc..2e1fd502 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,5 +1,7 @@ -grailsVersion=5.2.1 +grailsVersion=6.0.0 +grailsGradlePluginVersion=6.0.0 gormVersion=7.2.1 +org.gradle.caching=true org.gradle.daemon=true org.gradle.parallel=true org.gradle.jvmargs=-Dfile.encoding=UTF-8 -Xmx1024M @@ -10,4 +12,4 @@ webdriverBinariesVersion=2.6 chromeDriverVersion=2.45.0 geckodriverVersion=0.24.0 seleniumSafariDriverVersion=3.14.0 -alaSecurityLibsVersion=6.0.4 \ No newline at end of file +alaSecurityLibsVersion=6.2.0 diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index ffed3a25..98debb84 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.2-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-7.6.2-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew index 1b6c7873..65dcd68d 100755 --- a/gradlew +++ b/gradlew @@ -55,7 +55,7 @@ # Darwin, MinGW, and NonStop. # # (3) This script is generated from the Groovy template -# https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt # within the Gradle project. # # You can find Gradle at https://github.com/gradle/gradle/. @@ -80,10 +80,10 @@ do esac done -APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit - -APP_NAME="Gradle" +# This is normally unused +# shellcheck disable=SC2034 APP_BASE_NAME=${0##*/} +APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' @@ -143,12 +143,16 @@ fi if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then case $MAX_FD in #( max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC3045 MAX_FD=$( ulimit -H -n ) || warn "Could not query maximum file descriptor limit" esac case $MAX_FD in #( '' | soft) :;; #( *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC3045 ulimit -n "$MAX_FD" || warn "Could not set maximum file descriptor limit to $MAX_FD" esac @@ -205,6 +209,12 @@ set -- \ org.gradle.wrapper.GradleWrapperMain \ "$@" +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + # Use "xargs" to parse quoted args. # # With -n1 it outputs one arg per line, with the quotes and backslashes removed. diff --git a/gradlew.bat b/gradlew.bat index ac1b06f9..6689b85b 100755 --- a/gradlew.bat +++ b/gradlew.bat @@ -14,7 +14,7 @@ @rem limitations under the License. @rem -@if "%DEBUG%" == "" @echo off +@if "%DEBUG%"=="" @echo off @rem ########################################################################## @rem @rem Gradle startup script for Windows @@ -25,7 +25,8 @@ if "%OS%"=="Windows_NT" setlocal set DIRNAME=%~dp0 -if "%DIRNAME%" == "" set DIRNAME=. +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused set APP_BASE_NAME=%~n0 set APP_HOME=%DIRNAME% @@ -40,7 +41,7 @@ if defined JAVA_HOME goto findJavaFromJavaHome set JAVA_EXE=java.exe %JAVA_EXE% -version >NUL 2>&1 -if "%ERRORLEVEL%" == "0" goto execute +if %ERRORLEVEL% equ 0 goto execute echo. echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. @@ -75,13 +76,15 @@ set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar :end @rem End local scope for the variables with windows NT shell -if "%ERRORLEVEL%"=="0" goto mainEnd +if %ERRORLEVEL% equ 0 goto mainEnd :fail rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of rem the _cmd.exe /c_ return code! -if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 -exit /b 1 +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% :mainEnd if "%OS%"=="Windows_NT" endlocal diff --git a/grails-app/assets/javascripts/contribution.js b/grails-app/assets/javascripts/contribution.js new file mode 100644 index 00000000..8b27e73a --- /dev/null +++ b/grails-app/assets/javascripts/contribution.js @@ -0,0 +1,133 @@ +function instrument() { + var availableTags = [ + "institutionCode", + "collectionCode", + "catalogNumber", + "occurrenceID", + "recordNumber" + ]; + + function split( val ) { + return val.split( /,\s*/ ); + } + + function extractLast( term ) { + return split( term ).pop(); + } + + $( "input#termsForUniqueKey:enabled" ) + // don't navigate away from the field on tab when selecting an item + .bind( "keydown", function( event ) { + if ( event.keyCode === $.ui.keyCode.TAB && + $( this ).data( "autocomplete" ).menu.active ) { + event.preventDefault(); + } + }) + .autocomplete({ + minLength: 0, + source: function( request, response ) { + // delegate back to autocomplete, but extract the last term + response( $.ui.autocomplete.filter( + availableTags, extractLast( request.term ) ) ); + }, + focus: function() { + // prevent value inserted on focus + return false; + }, + select: function( event, ui ) { + var terms = split( this.value ); + // remove the current input + terms.pop(); + // add the selected item + terms.push( ui.item.value ); + // add placeholder to get the comma-and-space at the end + terms.push( "" ); + this.value = terms.join( ", " ); + return false; + } + }); +} + +function changeProtocol() { + var protocol = $('#protocolSelector').val(); + // remove autocomplete binding + // $('input#termsForUniqueKey:enabled').autocomplete('destroy'); + // $('input#termsForUniqueKey:enabled').unbind('keydown'); + // clear all + $('div.labile').css('display','none'); + $('div.labile input,textArea').attr('disabled','true'); + + // show the selected + console.log("Displaying protocol : " + protocol); + $.each(connectionParameters, function(key, obj) { + $.each(obj, function(j, p) { + if (p == protocol) { + $('div#connection_' + key).css('display','block'); + $('div#connection_' + key).removeAttr('style'); + $('div#connection_' + key + ' input,textArea').removeAttr('disabled'); + } + }) + }) + + // re-enable the autocomplete functionality + instrument(); +} + +instrument(); +//$('[name="start_date"]').datepicker({dateFormat: 'yy-mm-dd'}); +/* this expands lists of urls into an array of text inputs */ +// create a delete element that removes the element before it and itself +var $deleteLink = $(' ') + .click(function() { + $(this).prev().remove(); + $(this).remove(); + }); +// handle all urls (including hidden ones) +var urlInputs = $('input[name="url"]'); +$('input[name="url"]').addClass('input-xxlarge'); +$.each(urlInputs, function(i, obj) { + var urls = $(obj).val().split(','); + if (urls.length > 1) { + // more than one url so create an input for each extra one + $.each(urls,function(i,url) { + if (i == 0) { + // existing input gets the first url + $(obj).val(url); + } + else { + // clone the existing field and inject the next value - adding a delete link + $(obj).clone() + .val(url.trim()) + .css('width','93%') + .addClass('form-control') + .insertAfter($(obj).parent().children('input,span').last()) + .after($deleteLink.clone(true)); + } + }); + } +}); +/* this injects 'add another' functionality to urls */ +$.each(urlInputs, function(i, obj) { + $('Add another') + .insertAfter($(obj).parent().children('input,span').last()) + .click(function() { + // clone the original input + var $clone = $(obj).clone(); + $clone.val(''); + $clone.insertBefore(this); + $clone.after($deleteLink.clone(true)); // add delete link + }); +}); +/* this binds the code to add a new term to the list */ +$('#more-terms').click(function() { + var term = $('#otherKey').val(); + // check that term doesn't already exist + if ($('#'+term).length > 0) { + alert(term + " is already present"); + } + else { + var newField = "
" + + "
"; + $('#add-another').parent().append(newField); + } +}); diff --git a/grails-app/conf/application.yml b/grails-app/conf/application.yml index 8677a258..70f9bc72 100644 --- a/grails-app/conf/application.yml +++ b/grails-app/conf/application.yml @@ -178,7 +178,7 @@ security: - /css.* - /js.* - /static.* - appServerName: http://dev.ala.org.au:8080 + appServerName: http://localhost:8080 casServerUrlPrefix: https://auth-test.ala.org.au/cas loginUrl: https://auth-test.ala.org.au/cas/login logoutUrl: https://auth-test.ala.org.au/cas/logout @@ -400,7 +400,9 @@ environments: active: false dataSource: dbCreate: none - url: jdbc:mysql://localhost:3306/collectory?autoReconnect=true&connectTimeout=0&useUnicode=true&characterEncoding=UTF-8 + url: jdbc:mysql://localhost:3306/collectory?autoReconnect=true&connectTimeout=0&useUnicode=true&characterEncoding=UTF-8&serverTimezone=Australia/Sydney + username: specieslist + password: "specieslist" test: grails: assets: @@ -449,3 +451,7 @@ openapi: url: https://www.mozilla.org/en-US/MPL/1.1/ version: '@info.app.version@' cachetimeoutms: 0 + +sitemap: + dir: /data/collectory + enabled: true diff --git a/grails-app/conf/logback.xml b/grails-app/conf/logback.xml index c64e87a0..b739243f 100644 --- a/grails-app/conf/logback.xml +++ b/grails-app/conf/logback.xml @@ -11,7 +11,10 @@ + + + - \ No newline at end of file + diff --git a/grails-app/controllers/au/org/ala/collectory/AdminController.groovy b/grails-app/controllers/au/org/ala/collectory/AdminController.groovy index ac0a99f8..ba43e236 100644 --- a/grails-app/controllers/au/org/ala/collectory/AdminController.groovy +++ b/grails-app/controllers/au/org/ala/collectory/AdminController.groovy @@ -8,7 +8,7 @@ import org.springframework.core.io.support.PathMatchingResourcePatternResolver class AdminController { - def dataLoaderService, idGeneratorService, metadataService + def dataLoaderService, idGeneratorService, metadataService, sitemapService def index = { redirect(controller: 'manage') @@ -299,4 +299,10 @@ class AdminController { } return g } + + def buildSitemap() { + sitemapService.build() + + render text: 'done' + } } diff --git a/grails-app/controllers/au/org/ala/collectory/ProviderCodeController.groovy b/grails-app/controllers/au/org/ala/collectory/ProviderCodeController.groovy index 2aed73b4..d5dc668e 100644 --- a/grails-app/controllers/au/org/ala/collectory/ProviderCodeController.groovy +++ b/grails-app/controllers/au/org/ala/collectory/ProviderCodeController.groovy @@ -68,8 +68,8 @@ class ProviderCodeController { if (version != null) { if (providerCodeInstance.version > version) { providerCodeInstance.errors.rejectValue("version", "default.optimistic.locking.failure", - [message(code: 'providerCode.label', default: 'ProviderCode')] as Object[], - "Another user has updated this ProviderCode while you were editing") + [message(code: 'providerCode.label', default: 'ProviderCode')] as Object[], + "Another user has updated this ProviderCode while you were editing") render(view: "edit", model: [providerCodeInstance: providerCodeInstance]) return } @@ -89,23 +89,22 @@ class ProviderCodeController { @Transactional def delete(Long id) { if (collectoryAuthService?.userInRole(grailsApplication.config.ROLE_ADMIN)) { - def providerCodeInstance = ProviderCode.get(id) - if (!providerCodeInstance) { - flash.message = message(code: 'default.not.found.message', args: [message(code: 'providerCode.label', default: 'ProviderCode'), id]) - redirect(action: "list") - return - } + def providerCodeInstance = ProviderCode.get(id) + if (!providerCodeInstance) { + flash.message = message(code: 'default.not.found.message', args: [message(code: 'providerCode.label', default: 'ProviderCode'), id]) + redirect(action: "list") + return + } - try { - providerCodeInstance.delete(flush: true) - flash.message = message(code: 'default.deleted.message', args: [message(code: 'providerCode.label', default: 'ProviderCode'), id]) - redirect(action: "list") - } - catch (DataIntegrityViolationException e) { - flash.message = message(code: 'default.not.deleted.message', args: [message(code: 'providerCode.label', default: 'ProviderCode'), id]) - redirect(action: "show", id: id) - } - } else{ + try { + providerCodeInstance.delete(flush: true) + flash.message = message(code: 'default.deleted.message', args: [message(code: 'providerCode.label', default: 'ProviderCode'), id]) + redirect(action: "list") + } catch (DataIntegrityViolationException e) { + flash.message = message(code: 'default.not.deleted.message', args: [message(code: 'providerCode.label', default: 'ProviderCode'), id]) + redirect(action: "show", id: id) + } + } else { response.setHeader("Content-type", "text/plain; charset=UTF-8") render(message(code: "provider.group.controller.04", default: "You are not authorised to access this page.")) } diff --git a/grails-app/controllers/au/org/ala/collectory/ProviderGroupController.groovy b/grails-app/controllers/au/org/ala/collectory/ProviderGroupController.groovy index 510ad193..8d24d5f3 100644 --- a/grails-app/controllers/au/org/ala/collectory/ProviderGroupController.groovy +++ b/grails-app/controllers/au/org/ala/collectory/ProviderGroupController.groovy @@ -1,6 +1,6 @@ package au.org.ala.collectory - +import au.org.ala.collectory.resources.PP import grails.converters.JSON import org.springframework.dao.DataIntegrityViolationException import org.springframework.web.context.request.RequestContextHolder @@ -598,10 +598,46 @@ abstract class ProviderGroupController { } else { // are they allowed to edit if (isAuthorisedToEdit(pg.uid)) { + def connectionParams = metadataService.getConnectionParameters() + // set the default value for 'darwin core terms that uniquely identify a record' + try { + JSON.parse(pg.connectionParameters).each { k, v -> + + var item = connectionParams.find { ck, cv -> cv.paramName == k } + + if (item?.value) { + var defaultValue = v + if (item.value.paramName == 'termsForUniqueKey') { + defaultValue = v.join(',').replaceAll('"',"") + } else if (item.value.type == 'delimiter') { + def str = v + str = str.replaceAll('HT', PP.HT_CHAR) + str = str.replaceAll('LF', PP.LF_CHAR) + str = str.replaceAll('VT', PP.VT_CHAR) + str = str.replaceAll('FF', PP.FF_CHAR) + str = str.replaceAll('CR', PP.CR_CHAR) + defaultValue = str + } else if (item.value.paramName == 'url') { + if (v instanceof List) { + def normalised = [] + v.each { + if (it.trim().length() > 0) { + normalised << it.trim() + } + } + defaultValue = normalised.join(',').replaceAll('"',"") + } + } + + item.value.putAt('defaultValue', defaultValue) + } + } + } catch (ignored) { + } render(view:'upload', model:[ instance: pg, connectionProfiles: metadataService.getConnectionProfilesWithFileUpload(), - connectionParams: metadataService.getConnectionParameters() + connectionParams: connectionParams ]) } else { response.setHeader("Content-type", "text/plain; charset=UTF-8") @@ -870,7 +906,7 @@ abstract class ProviderGroupController { if (isAdmin()) { return true } else { - def email = RequestContextHolder.currentRequestAttributes()?.getUserPrincipal()?.name + def email = collectoryAuthService.authService.email ProviderGroup pg = providerGroupService?._get(uid) if (email && pg) { if(pg){ diff --git a/grails-app/controllers/au/org/ala/collectory/SitemapController.groovy b/grails-app/controllers/au/org/ala/collectory/SitemapController.groovy new file mode 100644 index 00000000..899cfc12 --- /dev/null +++ b/grails-app/controllers/au/org/ala/collectory/SitemapController.groovy @@ -0,0 +1,32 @@ +package au.org.ala.collectory + +class SitemapController { + + def index(Integer idx) { + if (!grailsApplication.config.sitemap.enabled) { + response.status = 404 + return + } + + File index = new File(grailsApplication.config.sitemap.dir + '/sitemap.xml') + if (!index.exists()) { + response.status = 404 + return + } + + response.contentType = "application/xml" + + if (idx == null) { + // return sitemap index + response.outputStream << index.newInputStream() + } else { + // return sitemap urls + File part = new File(grailsApplication.config.sitemap.dir + '/sitemap' + idx + ".xml") + if (!part.exists()) { + response.status = 404 + return + } + response.outputStream << part.newInputStream() + } + } +} diff --git a/grails-app/controllers/au/org/ala/collectory/UrlMappings.groovy b/grails-app/controllers/au/org/ala/collectory/UrlMappings.groovy index b4acfde0..3678512b 100644 --- a/grails-app/controllers/au/org/ala/collectory/UrlMappings.groovy +++ b/grails-app/controllers/au/org/ala/collectory/UrlMappings.groovy @@ -207,6 +207,8 @@ class UrlMappings { "/public/resources(.$format)"(controller: 'public', action: 'resources') "/public/condensed(.$format)"(controller: 'public', action: 'condensed') + "/sitemap($idx)?.xml"(controller: "sitemap", action: "index") + "/"(controller: 'public', action: 'map') "/error"(view: '/error') diff --git a/grails-app/i18n/messages.properties b/grails-app/i18n/messages.properties index 833c31f4..8aaf0d10 100644 --- a/grails-app/i18n/messages.properties +++ b/grails-app/i18n/messages.properties @@ -1276,6 +1276,7 @@ manage.list.addtools.des11=View and edit all known collection and institution co manage.list.addtools.mpm=Provider maps manage.list.addtools.des12=View and edit the allocation of collection and institution codes to collections manage.list.addtools.eadaj=Export all data as JSON +manage.list.addtools.buildsitemap=Build sitemap manage.list.addtools.des13=All tables exported verbatim as JSON manage.list.addtools.vae=Audit events manage.list.addtools.des14=All audit events diff --git a/grails-app/init/collectory/Application.groovy b/grails-app/init/collectory/Application.groovy index 5f87bfb0..b3284e36 100644 --- a/grails-app/init/collectory/Application.groovy +++ b/grails-app/init/collectory/Application.groovy @@ -4,10 +4,14 @@ import grails.boot.GrailsApp import grails.boot.config.GrailsAutoConfiguration import groovy.transform.CompileStatic +import org.springframework.context.annotation.ComponentScan +import org.springframework.scheduling.annotation.EnableScheduling +@ComponentScan('au.org.ala.collectory') @CompileStatic +@EnableScheduling class Application extends GrailsAutoConfiguration { static void main(String[] args) { GrailsApp.run(Application, args) } -} \ No newline at end of file +} diff --git a/grails-app/services/au/org/ala/collectory/CollectoryAuthService.groovy b/grails-app/services/au/org/ala/collectory/CollectoryAuthService.groovy index 2b983712..fa378b2f 100644 --- a/grails-app/services/au/org/ala/collectory/CollectoryAuthService.groovy +++ b/grails-app/services/au/org/ala/collectory/CollectoryAuthService.groovy @@ -41,13 +41,7 @@ class CollectoryAuthService{ static final API_KEY_COOKIE = "ALA-API-Key" def username() { - def username = 'not available' - if(RequestContextHolder.currentRequestAttributes()?.getUserPrincipal()?.name) - username = RequestContextHolder.currentRequestAttributes()?.getUserPrincipal()?.name - else { - if(authService) - username = authService.getUserName() - } + def username = authService.getDisplayName() return (username) ? username : 'not available' } diff --git a/grails-app/services/au/org/ala/collectory/ProviderGroupService.groovy b/grails-app/services/au/org/ala/collectory/ProviderGroupService.groovy index 60b3befd..9e283e0d 100644 --- a/grails-app/services/au/org/ala/collectory/ProviderGroupService.groovy +++ b/grails-app/services/au/org/ala/collectory/ProviderGroupService.groovy @@ -13,6 +13,7 @@ class ProviderGroupService { def collectoryAuthService def grailsApplication def messageSource + def authService def siteLocale = new Locale.Builder().setLanguageTag(Holders.config.siteDefaultLanguage as String).build() def serviceMethod() {} @@ -473,7 +474,7 @@ class ProviderGroupService { if (!grailsApplication.config.security.oidc.enabled.toBoolean() || isAdmin()) { return true } else { - def email = RequestContextHolder.currentRequestAttributes()?.getUserPrincipal()?.name + def email = authService.email if (email) { return _get(uid)?.isAuthorised(email) } diff --git a/grails-app/services/au/org/ala/collectory/SitemapService.groovy b/grails-app/services/au/org/ala/collectory/SitemapService.groovy new file mode 100644 index 00000000..6ff59561 --- /dev/null +++ b/grails-app/services/au/org/ala/collectory/SitemapService.groovy @@ -0,0 +1,127 @@ +/* + * Copyright (C) 2022 Atlas of Living Australia + * All Rights Reserved. + * + * The contents of this file are subject to the Mozilla Public + * License Version 1.1 (the "License"); you may not use this file + * except in compliance with the License. You may obtain a copy of + * the License at http://www.mozilla.org/MPL/ + * + * Software distributed under the License is distributed on an "AS + * IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or + * implied. See the License for the specific language governing + * rights and limitations under the License. + */ +package au.org.ala.collectory + +import org.springframework.scheduling.annotation.Scheduled + +import java.text.SimpleDateFormat + +class SitemapService { + + def grailsApplication + + + String URLSET_HEADER = "" + String URLSET_FOOTER = "" + + int MAX_URLS = 50000 // maximum number of URLs in a sitemap file + int MAX_SIZE = 9*1024*1024 // use 9MB to keep the actual file size below 10MB (a gateway limit) + + File currentFile + int fileCount = 0 + int countUrls = 0 + + SimpleDateFormat simpleDateFormat = new SimpleDateFormat("YYYY-MM-dd") + + FileWriter fw + + // run daily, initial delay 1hr + @Scheduled(fixedDelay = 86400000L, initialDelay = 3600000L) + def build() throws Exception { + initWriter() + buildSitemap() + closeWriter() + + buildSitemapIndex() + } + + def buildSitemapIndex() { + + // write parent sitemap file + fw = new FileWriter(grailsApplication.config.sitemap.dir + "/sitemap.xml") + fw.write("") + + for (int i=0;i" + grailsApplication.config.grails.serverURL + "/sitemap" + i + ".xml" + "") + fw.write("" + simpleDateFormat.format(new Date()) + "") + } + + fw.write("") + fw.flush() + fw.close() + } + + def initWriter() { + currentFile = new File(grailsApplication.config.sitemap.dir + "/sitemap" + fileCount + ".xml.tmp") + + fw = new FileWriter(currentFile) + + fw.write(URLSET_HEADER) + + countUrls = 0 + fileCount++ + } + + def closeWriter() { + fw.write(URLSET_FOOTER) + fw.flush() + fw.close() + } + + def writeUrl(Date lastUpdated, String changefreq, String encodedUrl) { + if (countUrls >= MAX_URLS || currentFile.size() >= MAX_SIZE) { + closeWriter() + initWriter() + } + + fw.write("") + fw.write("" + encodedUrl + "") + fw.write("" + simpleDateFormat.format(lastUpdated) + "") + fw.write("" + changefreq + "") + fw.write("") + + fw.flush() + + countUrls++ + } + + def buildSitemap() throws Exception { + + Collection.findAll().each {Collection it -> + writeUrl(it.lastUpdated, "weekly", grailsApplication.config.grails.serverURL + "/public/show/co" + it.id) + } + + Institution.findAll().each {Institution it -> + writeUrl(it.lastUpdated, "weekly", grailsApplication.config.grails.serverURL + "/public/show/in" + it.id) + } + + DataProvider.findAll().each {DataProvider it -> + writeUrl(it.lastUpdated, "weekly", grailsApplication.config.grails.serverURL + "/public/show/dp" + it.id) + } + + DataResource.findAllByIsPrivate(false).each {DataResource it -> + writeUrl(it.lastUpdated, "weekly", grailsApplication.config.grails.serverURL + "/public/show/dr" + it.id) + } + } +} diff --git a/grails-app/taglib/au/org/ala/collectory/CollectoryTagLib.groovy b/grails-app/taglib/au/org/ala/collectory/CollectoryTagLib.groovy index 9ce1cd55..e93267bf 100644 --- a/grails-app/taglib/au/org/ala/collectory/CollectoryTagLib.groovy +++ b/grails-app/taglib/au/org/ala/collectory/CollectoryTagLib.groovy @@ -14,7 +14,7 @@ import java.text.SimpleDateFormat class CollectoryTagLib { - def collectoryAuthService, metadataService, providerGroupService + def collectoryAuthService, metadataService, providerGroupService, authService static namespace = 'cl' @@ -196,7 +196,7 @@ class CollectoryTagLib { * @attrs uid - the uid of the entity */ def isAuth = { attrs, body -> - if (isAuthorisedToEdit(attrs.uid, request.getRemoteUser())) { + if (isAuthorisedToEdit(attrs.uid, authService.email)) { out << body() } else { out << ' You are not authorised to change this record '// + debugString @@ -1536,7 +1536,7 @@ class CollectoryTagLib { * @body the label for the button - defaults to 'Edit' if not specified */ def editButton = { attrs, body -> - if (isAuthorisedToEdit(attrs.uid, request.getRemoteUser())) { + if (isAuthorisedToEdit(attrs.uid, authService.email)) { def paramsMap // anchor class paramsMap = [class:'edit btn btn-default'] @@ -1833,6 +1833,9 @@ class CollectoryTagLib { onchange:'changeProtocol()') out << "" + // map each parameter only once + var mappedParams = [:] + // create the widgets for each protocol (profile) metadataService.getConnectionProfilesAsList().each { // is this the selected protocol? @@ -1842,62 +1845,72 @@ class CollectoryTagLib { it.params.each { ppName -> def pp = metadataService.getConnectionParameter(ppName) + if (mappedParams[pp.paramName] != null) { + mappedParams[pp.paramName].add(it.name) + } else { + mappedParams[pp.paramName] = [it.name] - // get value from object - def displayedValue = cp?."${pp.paramName}"?:"" + // get value from object + def displayedValue = cp?."${pp.paramName}" ?: "" - // inject default if no value - if (!displayedValue && pp.defaultValue) { - displayedValue = pp.defaultValue - } + // inject default if no value + if (!displayedValue && pp.defaultValue) { + displayedValue = pp.defaultValue + } - // unravel any JSON lists - if (displayedValue instanceof JSONArray) { - displayedValue = displayedValue.collect {it}.join(', ') as String - } + // unravel any JSON lists + if (displayedValue instanceof JSONArray) { + displayedValue = displayedValue.collect { it }.join(', ') as String + } - // handle unprintable chars - if (pp.type == 'delimiter') { - displayedValue = encodeControlChars(displayedValue) - } + // handle unprintable chars + if (pp.type == 'delimiter') { + displayedValue = encodeControlChars(displayedValue) + } - def attributes = [name:pp.paramName, value:displayedValue, class:'form-control'] - if (!selected) { - attributes << [disabled:true] - } - if (pp.paramName == "termsForUniqueKey") { - // handle terms specially - out << """
""" - out << """
Don't change the following terms unless you know what you are doing. Incorrect values can cause major devastation.
""" - out << "
" - out << """
""" - out << """""" - out << textField(attributes) - out << "
" - } else if (pp.type == 'boolean') { - attributes.remove('class') - out << """
""" - out << """
" - } else { - // all others - def widget - switch (pp.type) { - case 'textArea': widget = 'textArea'; break - default: widget = 'textField'; break + def attributes = [name: pp.paramName, value: displayedValue, class: 'form-control'] + if (!selected) { + attributes << [disabled: true] + } + if (pp.paramName == "termsForUniqueKey") { + // handle terms specially + out << """
""" + out << """
Don't change the following terms unless you know what you are doing. Incorrect values can cause major devastation.
""" + out << "
" + out << """
""" + out << """""" + out << textField(attributes) + out << "
" + } else if (pp.type == 'boolean') { + attributes.remove('class') + out << """
""" + out << """
" + } else { + // all others + def widget + switch (pp.type) { + case 'textArea': widget = 'textArea'; break + default: widget = 'textField'; break + } + out << """
""" + out << """""" + out << "${widget}"(attributes) + out << "
" } - out << """
""" - out << """""" - out << "${widget}"(attributes) - out << "
" } } } + out << "" + } def lastChecked = { attrs -> diff --git a/grails-app/views/dataResource/contribution.gsp b/grails-app/views/dataResource/contribution.gsp index 0ea2d221..37bf7c41 100644 --- a/grails-app/views/dataResource/contribution.gsp +++ b/grails-app/views/dataResource/contribution.gsp @@ -13,6 +13,7 @@ }; +
@@ -139,135 +140,5 @@
- - function instrument() { - var availableTags = [ - "institutionCode", - "collectionCode", - "catalogNumber", - "occurrenceID", - "recordNumber" - ]; - - function split( val ) { - return val.split( /,\s*/ ); - } - - function extractLast( term ) { - return split( term ).pop(); - } - - $( "input#termsForUniqueKey:enabled" ) - // don't navigate away from the field on tab when selecting an item - .bind( "keydown", function( event ) { - if ( event.keyCode === $.ui.keyCode.TAB && - $( this ).data( "autocomplete" ).menu.active ) { - event.preventDefault(); - } - }) - .autocomplete({ - minLength: 0, - source: function( request, response ) { - // delegate back to autocomplete, but extract the last term - response( $.ui.autocomplete.filter( - availableTags, extractLast( request.term ) ) ); - }, - focus: function() { - // prevent value inserted on focus - return false; - }, - select: function( event, ui ) { - var terms = split( this.value ); - // remove the current input - terms.pop(); - // add the selected item - terms.push( ui.item.value ); - // add placeholder to get the comma-and-space at the end - terms.push( "" ); - this.value = terms.join( ", " ); - return false; - } - }); - } - - function changeProtocol() { - var protocol = $('#protocolSelector').val(); - // remove autocomplete binding - $('input#termsForUniqueKey:enabled').autocomplete('destroy'); - $('input#termsForUniqueKey:enabled').unbind('keydown'); - // clear all - $('div.labile').css('display','none'); - $('div.labile input,textArea').attr('disabled','true'); - - // show the selected - console.log("Displaying protocol: " + protocol); - $('div#' + protocol).css('display','block'); - $('div#' + protocol).removeAttr('style'); - $('div#' + protocol + ' input,textArea').removeAttr('disabled'); - - // re-enable the autocomplete functionality - instrument(); - } - - instrument(); - //$('[name="start_date"]').datepicker({dateFormat: 'yy-mm-dd'}); - /* this expands lists of urls into an array of text inputs */ - // create a delete element that removes the element before it and itself - %{--var deleteImageUrl = "${resource(dir:'/images/ala',file:'delete.png')}";--}% - var $deleteLink = $(' ') - .click(function() { - $(this).prev().remove(); - $(this).remove(); - }); - // handle all urls (including hidden ones) - var urlInputs = $('input[name="url"]'); - $('input[name="url"]').addClass('input-xxlarge'); - $.each(urlInputs, function(i, obj) { - var urls = $(obj).val().split(','); - if (urls.length > 1) { - // more than one url so create an input for each extra one - $.each(urls,function(i,url) { - if (i == 0) { - // existing input gets the first url - $(obj).val(url); - } - else { - // clone the existing field and inject the next value - adding a delete link - $(obj).clone() - .val(url.trim()) - .css('width','93%') - .addClass('form-control') - .insertAfter($(obj).parent().children('input,span').last()) - .after($deleteLink.clone(true)); - } - }); - } - }); - /* this injects 'add another' functionality to urls */ - $.each(urlInputs, function(i, obj) { - $('Add another') - .insertAfter($(obj).parent().children('input,span').last()) - .click(function() { - // clone the original input - var $clone = $(obj).clone(); - $clone.val(''); - $clone.insertBefore(this); - $clone.after($deleteLink.clone(true)); // add delete link - }); - }); - /* this binds the code to add a new term to the list */ - $('#more-terms').click(function() { - var term = $('#otherKey').val(); - // check that term doesn't already exist - if ($('#'+term).length > 0) { - alert(term + " is already present"); - } - else { - var newField = "
" + - "
"; - $('#add-another').parent().append(newField); - } - }); - - \ No newline at end of file + diff --git a/grails-app/views/manage/index.gsp b/grails-app/views/manage/index.gsp index 982f04b9..0429e3c8 100644 --- a/grails-app/views/manage/index.gsp +++ b/grails-app/views/manage/index.gsp @@ -7,10 +7,10 @@ /> - +
- +
@@ -51,4 +51,4 @@
- \ No newline at end of file + diff --git a/grails-app/views/manage/list.gsp b/grails-app/views/manage/list.gsp index b6e84399..dbfb3df9 100644 --- a/grails-app/views/manage/list.gsp +++ b/grails-app/views/manage/list.gsp @@ -237,6 +237,18 @@ %{-- --}% + +
+
+ +
+ + %{--
--}% + %{-- --}% + %{--

--}% + %{--
--}% + +

Data sync with GBIF

diff --git a/grails-app/views/providerCode/list.gsp b/grails-app/views/providerCode/list.gsp index a8553b3a..b74c37f4 100644 --- a/grails-app/views/providerCode/list.gsp +++ b/grails-app/views/providerCode/list.gsp @@ -12,7 +12,7 @@
    • -
    • +
@@ -30,9 +30,9 @@ - + ${fieldValue(bean: providerCodeInstance, field: "code")} - + diff --git a/grails-app/views/providerMap/_form.gsp b/grails-app/views/providerMap/_form.gsp index 475f01e2..44409286 100644 --- a/grails-app/views/providerMap/_form.gsp +++ b/grails-app/views/providerMap/_form.gsp @@ -12,12 +12,12 @@
- + - + - +
@@ -118,4 +118,4 @@ $('#institutionSelect').change(function() { }); } }); - \ No newline at end of file + diff --git a/grails-app/views/providerMap/list.gsp b/grails-app/views/providerMap/list.gsp index 803a245f..9a1be1f6 100644 --- a/grails-app/views/providerMap/list.gsp +++ b/grails-app/views/providerMap/list.gsp @@ -14,6 +14,7 @@
    • +
@@ -27,7 +28,7 @@ - + @@ -35,33 +36,33 @@ - + - + - + - + - + - + - + - + - + - + diff --git a/grails-app/views/shared/contactRole.gsp b/grails-app/views/shared/contactRole.gsp index 764deb4b..0faf0600 100644 --- a/grails-app/views/shared/contactRole.gsp +++ b/grails-app/views/shared/contactRole.gsp @@ -30,21 +30,21 @@ - + - + - + - + diff --git a/grails-wrapper.jar b/grails-wrapper.jar index 6b6da64f..bc85146c 100644 Binary files a/grails-wrapper.jar and b/grails-wrapper.jar differ diff --git a/grailsw b/grailsw index 8d0cc123..c2c921c2 100755 --- a/grailsw +++ b/grailsw @@ -102,7 +102,6 @@ fi if $cygwin ; then APP_HOME=`cygpath --path --mixed "$APP_HOME"` JAVACMD=`cygpath --unix "$JAVACMD"` - JAR_PATH=`cygpath --path --mixed "$JAR_PATH"` # We build the pattern for arguments to be converted via cygpath ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` diff --git a/settings.gradle b/settings.gradle new file mode 100644 index 00000000..464e7af9 --- /dev/null +++ b/settings.gradle @@ -0,0 +1,20 @@ +pluginManagement { + repositories { + mavenLocal() + maven { url "https://nexus.ala.org.au/content/groups/public/" } + maven { url "https://repo.grails.org/grails/core/" } + gradlePluginPortal() + } + plugins { + id "org.grails.grails-web" version "6.0.0" + id "org.grails.grails-gsp" version "6.0.0" + id "com.bertramlabs.asset-pipeline" version "4.3.0" +// classpath "org.grails:grails-gradle-plugin:$grailsVersion" +// classpath "org.grails.plugins:hibernate5:${gormVersion}" +// classpath "gradle.plugin.com.github.erdi.webdriver-binaries:webdriver-binaries-gradle-plugin:$webdriverBinariesVersion" +// classpath "com.bertramlabs.plugins:asset-pipeline-gradle:3.4.6" +// classpath 'org.grails.plugins:database-migration:4.2.0' + } +} + +rootProject.name='collectory'
Institution CodesCollection Codes
${fieldValue(bean: providerMapInstance, field: "id")} ${fieldValue(bean: providerMapInstance, field: "institution")} ${fieldValue(bean: providerMapInstance, field: "collection")}${providerMapInstance.getInstitutionCodes().join(' ')}${providerMapInstance.getCollectionCodes().join(' ')}
${cf.contact?.buildName()} for ${command.name}
:
${entityNameLower}.
${entityNameLower}.
: ${entityNameLower}.