Merge pull request #301 from AtlasOfLivingAustralia/290-add-optional-…

…s3-usage

#290 Added optional S3 data store for BS CSV files

build.gradle

@@ -54,8 +54,8 @@ bootWar {
 repositories {
  repositories {
  mavenLocal()
- maven { url "https://nexus.ala.org.au/content/groups/public/" }
  mavenCentral()
+ maven { url "https://nexus.ala.org.au/content/groups/public/" }
  maven { url "https://repo.grails.org/grails/core/" }
  }
  mavenCentral()
@@ -127,6 +127,8 @@ dependencies {
  implementation "org.grails.plugins:ala-ws-plugin:$alaSecurityLibsVersion"
  implementation "au.org.ala:userdetails-service-client:$alaSecurityLibsVersion"
  implementation "org.grails.plugins:ala-admin-plugin:2.3.0"
+ implementation "au.org.ala.plugins:openapi:1.3.0"
+ implementation "org.grails.plugins:aws-sdk-s3:2.4.8"
 
  // for ContentType
  implementation "org.apache.httpcomponents:httpcore:4.4.16"
@@ -136,7 +138,7 @@ dependencies {
  //Using AWS SES as email provider
  implementation 'org.grails.plugins:mail:4.0.0'
  implementation 'au.org.ala:ala-mail:1.0.0-SNAPSHOT'
- implementation group: 'com.amazonaws', name: 'aws-java-sdk-sts', version: "1.12.759"
+ implementation 'com.amazonaws:aws-java-sdk-sts:1.12.759'
 
  implementation 'org.grails.plugins:schwartz-monitor:2.0.1.ALA-SNAPSHOT'
  implementation 'org.grails.plugins:quartz:2.0.13'
@@ -162,7 +164,6 @@ dependencies {
  runtimeOnly 'xalan:xalan:2.7.3'
  implementation 'com.jayway.jsonpath:json-path:2.7.0'
  implementation 'com.jayway.jsonpath:json-path-assert:2.7.0'
- implementation 'au.org.ala.plugins:openapi:1.3.0'
 }
 //
 //bootRun {

grails-app/conf/application.yml

@@ -120,6 +120,14 @@ grails:
  scriptlets: html
  taglib: none
  staticparts: none
+ plugin:
+ awssdk:
+ region: ap-southeast-2
+ s3:
+# # accessKey: # Not required on EC2 deployments
+# # secretKey: # Not required on EC2 deployments
+# bucket: ala-alerts-test
+# profile: comp-ala-developer
 ---
 headerAndFooter:
  baseURL: "https://www.ala.org.au/commonui-bs3-2019"
@@ -179,11 +187,12 @@ biosecurity:
  link: https://www.ala.org.au/blogs-news/ala-helps-to-stop-pests-in-their-tracks/
  csv:
  local:
- enabled: true
+ enabled: false # See environments.development section below where set to true
  directory: /data/alerts/csv
  s3:
- enabled: false
- bucket: alerts
+ enabled: true
+ directory: biosecurity # key prefix
+ # bucket: alerts # configured via plugin config `grails.plugin.awssdk.s3.bucket`
  # read lga from the layer
  lga: LGA2023
  subscriptionsPerPage: 100
@@ -211,6 +220,15 @@ environments:
  host: "localhost"
  port: 1025
  #username: postie.emailSender
+ plugin:
+ awssdk:
+ region: ap-southeast-2
+ s3:
+ # accessKey:
+ # secretKey:
+ bucket: ala-alerts-test
+ profile: comp-ala-developer
+ region: ap-southeast-2
  security:
  cas:
  appServerName: "http://localhost:8080"
@@ -220,6 +238,8 @@ environments:
  csv:
  local:
  enabled: true
+ s3:
+ enabled: false
  userdetails:
  web:
  url: https://aws-auth-test-2023.test.ala.org.au/userdetails/

grails-app/conf/logback.xml

@@ -14,8 +14,9 @@
  <logger name="org.hibernate.orm" level="ERROR" />
  <logger name="org.grails.config.NavigableMap" level="ERROR" />
  <logger name="org.quartz" level="ERROR" />
-<!-- <logger name="au.org.ala" level="DEBUG" />-->
+ <logger name="au.org.ala.bootstrap3" level="INFO" />
 
+ <logger name="au.org.ala" level="DEBUG" />
  <logger name="au.org.ala.cas.client" level="WARN">
  <appender-ref ref="STDOUT" />
  </logger>

grails-app/controllers/au/org/ala/alerts/AdminController.groovy

@@ -716,16 +716,20 @@ class AdminController {
 
  @AlaSecured(value = ['ROLE_ADMIN', 'ROLE_BIOSECURITY_ADMIN'], anyRole = true, redirectController = 'notification', redirectAction = 'myAlerts', message = "You don't have permission to view that page.")
  def listBiosecurityAuditCSV() {
- def result = biosecurityCSVService.list()
+ def result = [:]
+ try {
+ result = biosecurityCSVService.list()
+ } catch (Exception e) {
+ log.error("Error in listing Biosecurity CSV files: ${e.message}")
+ result = [status: 1, message: "Error in listing Biosecurity CSV files: ${e.message}"]
+ }
  render(view: 'biosecurityCSV', model: result)
  }
 
  @AlaSecured(value = ['ROLE_ADMIN', 'ROLE_BIOSECURITY_ADMIN'], anyRole = true,redirectController = 'notification', redirectAction = 'myAlerts', message = "You don't have permission to view that page.")
  def aggregateBiosecurityAuditCSV(String folderName) {
- def BASE_DIRECTORY = grailsApplication.config.biosecurity.csv.local.directory
- def folder = new File(BASE_DIRECTORY, folderName)
- if (!folder.exists() || !folder.isDirectory()) {
- render(status: 404, text: "Data not found")
+ if (!biosecurityCSVService.folderExists(folderName)) {
+ render(status: 404, text: 'Data not found')
  return
  }
 
@@ -742,17 +746,17 @@ class AdminController {
 
  @AlaSecured(value = ['ROLE_ADMIN', 'ROLE_BIOSECURITY_ADMIN'], anyRole = true,redirectController = 'notification', redirectAction = 'myAlerts', message = "You don't have permission to view that page.")
  def downloadBiosecurityAuditCSV(String filename) {
- def BASE_DIRECTORY = grailsApplication.config.biosecurity.csv.local.directory
- def file = new File(BASE_DIRECTORY, filename)
- if (!file.exists() || file.isDirectory()) {
+ String contents = biosecurityCSVService.getFile(filename)
+ if (!contents) {
  render(status: 404, text: "Data not found")
  return
  }
-
- def saveToFile = Paths.get(filename).collect { it.toString() }.join("_")
- response.contentType = 'application/octet-stream'
- response.setHeader('Content-Disposition', "attachment; filename=\"${saveToFile}\"")
- response.outputStream << file.bytes
+ response.contentType = 'text/csv'
+ response.setHeader("Content-disposition", "attachment; filename=\"${filename.tokenize(File.separator).last()}\"")
+ response.outputStream.withWriter("UTF-8") { writer ->
+ writer << contents
+ }
+ response.outputStream.flush()
  }
 
  /**
@@ -781,22 +785,14 @@ class AdminController {
 
  @AlaSecured(value = ['ROLE_ADMIN', 'ROLE_BIOSECURITY_ADMIN'], anyRole = true)
  def deleteBiosecurityAuditCSV(String filename) {
- Map message = [status : 1, message: ""]
- def BASE_DIRECTORY = grailsApplication.config.biosecurity.csv.local.directory
- def file = new File(BASE_DIRECTORY, filename)
- if (!file.exists() || file.isDirectory()) {
- message['status'] = 1
- message['message'] = "File not found"
- } else {
- if (file.renameTo(new File(BASE_DIRECTORY, filename +'_deleted'))){
- message['status'] = 0
- message['message'] = "The file has been temporarily deleted. You can contact the system administrator to recover it."
- } else {
- message['status'] = 1
- message['message'] = "File deletion failed"
- }
- }
+ Map message = biosecurityCSVService.deleteFile(filename)
+ render(status: 200, contentType: 'application/json', text: message as JSON)
+ }
 
+ @AlaSecured(value = ['ROLE_ADMIN', 'ROLE_BIOSECURITY_ADMIN'], anyRole = true)
+ def moveLocalFilesToS3() {
+ Boolean dryRun = params.boolean('dryRun', true)
+ Map message = biosecurityCSVService.moveLocalFilesToS3(dryRun)
  render(status: 200, contentType: 'application/json', text: message as JSON)
  }
 }