#290 Added optional S3 data store for BS CSV files

build.gradle

@@ -127,6 +127,8 @@ dependencies {
     implementation "org.grails.plugins:ala-ws-plugin:$alaSecurityLibsVersion"
     implementation "au.org.ala:userdetails-service-client:$alaSecurityLibsVersion"
     implementation "org.grails.plugins:ala-admin-plugin:2.3.0"
+    implementation "au.org.ala.plugins:openapi:1.3.0"
+    implementation "org.grails.plugins:aws-sdk-s3:2.4.8"
 
     // for ContentType
     implementation "org.apache.httpcomponents:httpcore:4.4.16"
@@ -136,7 +138,7 @@ dependencies {
     //Using AWS SES as email provider
     implementation 'org.grails.plugins:mail:4.0.0'
     implementation 'au.org.ala:ala-mail:1.0.0-SNAPSHOT'
-    implementation group: 'com.amazonaws', name: 'aws-java-sdk-sts', version: "1.12.759"
+    implementation 'com.amazonaws:aws-java-sdk-sts:1.12.759'
 
     implementation 'org.grails.plugins:schwartz-monitor:2.0.1.ALA-SNAPSHOT'
     implementation 'org.grails.plugins:quartz:2.0.13'
@@ -162,7 +164,6 @@ dependencies {
     runtimeOnly 'xalan:xalan:2.7.3'
     implementation 'com.jayway.jsonpath:json-path:2.7.0'
     implementation 'com.jayway.jsonpath:json-path-assert:2.7.0'
-    implementation 'au.org.ala.plugins:openapi:1.3.0'
 }
 //
 //bootRun {

grails-app/conf/application.yml

@@ -179,11 +179,12 @@ biosecurity:
         link: https://www.ala.org.au/blogs-news/ala-helps-to-stop-pests-in-their-tracks/
     csv:
         local:
-            enabled: true
+            enabled: false # See environments.development section below where set to true
             directory: /data/alerts/csv
         s3:
-            enabled: false
-            bucket: alerts
+            enabled: true
+            directory: biosecurity # key prefix
+            # bucket: alerts # configured via plugin config `grails.plugin.awssdk.s3.bucket`
         # read lga from the layer
         lga: LGA2023
     subscriptionsPerPage: 100
@@ -203,6 +204,8 @@ biosecurity:
         eventDateAge: 150
         firstLoadedDateAge: 8
 
+
+
 environments:
     development:
         grails:
@@ -211,6 +214,15 @@ environments:
                 host: "localhost"
                 port: 1025
                 #username: postie.emailSender
+            plugin:
+                awssdk:
+                    region: ap-southeast-2
+                    s3:
+                        #          accessKey:
+                        #          secretKey:
+                        bucket: ala-alerts-test
+                        profile: comp-ala-developer
+                        region: ap-southeast-2
         security:
             cas:
                 appServerName: "http://localhost:8080"
@@ -220,6 +232,8 @@ environments:
             csv:
                 local:
                     enabled: true
+                s3:
+                    enabled: false
         userdetails:
             web:
                 url: https://aws-auth-test-2023.test.ala.org.au/userdetails/

grails-app/conf/logback.xml

@@ -14,8 +14,9 @@
     <logger name="org.hibernate.orm" level="ERROR" />
     <logger name="org.grails.config.NavigableMap" level="ERROR" />
     <logger name="org.quartz" level="ERROR" />
-<!--    <logger name="au.org.ala" level="DEBUG" />-->
+    <logger name="au.org.ala.bootstrap3" level="INFO" />
 
+    <logger name="au.org.ala" level="DEBUG" />
     <logger name="au.org.ala.cas.client" level="WARN">
         <appender-ref ref="STDOUT" />
     </logger>

grails-app/controllers/au/org/ala/alerts/AdminController.groovy

@@ -716,16 +716,20 @@ class AdminController {
 
     @AlaSecured(value = ['ROLE_ADMIN', 'ROLE_BIOSECURITY_ADMIN'], anyRole = true, redirectController = 'notification', redirectAction = 'myAlerts', message = "You don't have permission to view that page.")
     def listBiosecurityAuditCSV() {
-        def result  = biosecurityCSVService.list()
+        def result = [:]
+        try {
+            result  = biosecurityCSVService.list()
+        } catch (Exception e) {
+            log.error("Error in listing Biosecurity CSV files: ${e.message}")
+            result = [status: 1, message: "Error in listing Biosecurity CSV files: ${e.message}"]
+        }
         render(view: 'biosecurityCSV', model: result)
     }
 
     @AlaSecured(value = ['ROLE_ADMIN', 'ROLE_BIOSECURITY_ADMIN'], anyRole = true,redirectController = 'notification', redirectAction = 'myAlerts', message = "You don't have permission to view that page.")
     def aggregateBiosecurityAuditCSV(String folderName) {
-        def BASE_DIRECTORY = grailsApplication.config.biosecurity.csv.local.directory
-        def folder = new File(BASE_DIRECTORY, folderName)
-        if (!folder.exists() || !folder.isDirectory()) {
-            render(status: 404, text: "Data not found")
+        if (!biosecurityCSVService.folderExists(folderName)) {
+            render(status: 404, text: 'Data not found')
             return
         }
 
@@ -742,17 +746,17 @@ class AdminController {
 
     @AlaSecured(value = ['ROLE_ADMIN', 'ROLE_BIOSECURITY_ADMIN'], anyRole = true,redirectController = 'notification', redirectAction = 'myAlerts', message = "You don't have permission to view that page.")
     def downloadBiosecurityAuditCSV(String filename) {
-        def BASE_DIRECTORY = grailsApplication.config.biosecurity.csv.local.directory
-        def file = new File(BASE_DIRECTORY, filename)
-        if (!file.exists() || file.isDirectory()) {
+        String contents = biosecurityCSVService.getFile(filename)
+        if (!contents) {
             render(status: 404, text: "Data not found")
             return
         }
-
-        def saveToFile = Paths.get(filename).collect { it.toString() }.join("_")
-        response.contentType = 'application/octet-stream'
-        response.setHeader('Content-Disposition', "attachment; filename=\"${saveToFile}\"")
-        response.outputStream << file.bytes
+        response.contentType = 'text/csv'
+        response.setHeader("Content-disposition", "attachment; filename=\"${filename.tokenize(File.separator).last()}\"")
+        response.outputStream.withWriter("UTF-8") { writer ->
+            writer << contents
+        }
+        response.outputStream.flush()
     }
 
     /**
@@ -781,22 +785,14 @@ class AdminController {
 
     @AlaSecured(value = ['ROLE_ADMIN', 'ROLE_BIOSECURITY_ADMIN'], anyRole = true)
     def deleteBiosecurityAuditCSV(String filename) {
-        Map message = [status : 1, message: ""]
-        def BASE_DIRECTORY = grailsApplication.config.biosecurity.csv.local.directory
-        def file = new File(BASE_DIRECTORY, filename)
-        if (!file.exists() || file.isDirectory()) {
-            message['status'] = 1
-            message['message'] = "File not found"
-        } else {
-            if (file.renameTo(new File(BASE_DIRECTORY,  filename +'_deleted'))){
-                message['status'] = 0
-                message['message'] = "The file has been temporarily deleted. You can contact the system administrator to recover it."
-            } else {
-                message['status'] = 1
-                message['message'] = "File deletion failed"
-            }
-        }
+        Map message = biosecurityCSVService.deleteFile(filename)
+        render(status: 200, contentType: 'application/json', text: message as JSON)
+    }
 
+    @AlaSecured(value = ['ROLE_ADMIN', 'ROLE_BIOSECURITY_ADMIN'], anyRole = true)
+    def moveLocalFilesToS3() {
+        Boolean dryRun = params.boolean('dryRun', true)
+        Map message = biosecurityCSVService.moveLocalFilesToS3(dryRun)
         render(status: 200, contentType: 'application/json', text: message as JSON)
     }
 }