Add support to read configuration resources from S3 for Deposit Services

pass-deposit-services/deposit-core/pom.xml

@@ -35,8 +35,21 @@
     <greenmail-junit5.version>2.0.1</greenmail-junit5.version>
     <javax-json.version>1.1.4</javax-json.version>
     <maven-model.version>3.9.6</maven-model.version>
+    <awsspring.version>3.1.0</awsspring.version>
   </properties>
 
+  <dependencyManagement>
+    <dependencies>
+      <dependency>
+        <groupId>io.awspring.cloud</groupId>
+        <artifactId>spring-cloud-aws-dependencies</artifactId>
+        <version>${awsspring.version}</version>
+        <type>pom</type>
+        <scope>import</scope>
+      </dependency>
+    </dependencies>
+  </dependencyManagement>
+
   <dependencies>
 
     <dependency>
@@ -64,6 +77,11 @@
       <artifactId>spring-integration-mail</artifactId>
     </dependency>
 
+    <dependency>
+      <groupId>io.awspring.cloud</groupId>
+      <artifactId>spring-cloud-aws-starter-s3</artifactId>
+    </dependency>
+
     <dependency>
       <groupId>org.jsoup</groupId>
       <artifactId>jsoup</artifactId>

pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/config/spring/RepositoriesFactoryBeanConfig.java

@@ -32,9 +32,6 @@ public class RepositoriesFactoryBeanConfig {
 
     private static final Logger LOG = LoggerFactory.getLogger(RepositoriesFactoryBeanConfig.class);
 
-    @Value("${pass.deposit.repository.configuration}")
-    private Resource repositoryConfigResource;
-
     @Bean
     public ObjectMapper repositoriesMapper(Environment env) {
         ObjectMapper mapper = new ObjectMapper();

pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/config/spring/RepositoriesConfigS3IT.java

@@ -0,0 +1,107 @@
+/*
+ * Copyright 2024 Johns Hopkins University
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.eclipse.pass.deposit.config.spring;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.testcontainers.containers.localstack.LocalStackContainer.Service.S3;
+
+import java.io.IOException;
+
+import org.eclipse.pass.deposit.DepositApp;
+import org.eclipse.pass.deposit.config.repository.Repositories;
+import org.eclipse.pass.deposit.config.repository.RepositoryConfig;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.test.context.DynamicPropertyRegistry;
+import org.springframework.test.context.DynamicPropertySource;
+import org.springframework.test.context.TestPropertySource;
+import org.springframework.test.context.junit.jupiter.SpringExtension;
+import org.testcontainers.containers.BindMode;
+import org.testcontainers.containers.localstack.LocalStackContainer;
+import org.testcontainers.junit.jupiter.Container;
+import org.testcontainers.junit.jupiter.Testcontainers;
+import org.testcontainers.utility.DockerImageName;
+
+/**
+ * @author Russ Poetker ([email protected])
+ */
+@ExtendWith(SpringExtension.class)
+@SpringBootTest(classes = DepositApp.class)
+@TestPropertySource(properties = {
+    "spring.jms.listener.auto-startup=false",
+    "spring.cloud.aws.s3.enabled=true",
+    "pass.deposit.repository.configuration=s3://test-bucket/s3-test-repositories.json"
+})
+@Testcontainers
+public class RepositoriesConfigS3IT {
+
+    private static final DockerImageName LOCALSTACK_IMG =
+        DockerImageName.parse("localstack/localstack:3.1.0");
+
+    @Container
+    static final LocalStackContainer localStack =
+        new LocalStackContainer(LOCALSTACK_IMG)
+            .withClasspathResourceMapping("/full-test-s3-repositories.json",
+                "/tmp/tmp-test-repositories.json", BindMode.READ_ONLY)
+            .withServices(S3);
+
+    @Autowired
+    private Repositories repositories;
+
+    @DynamicPropertySource
+    static void overrideProperties(DynamicPropertyRegistry registry) {
+        registry.add("spring.cloud.aws.region.static", localStack::getRegion);
+        registry.add("spring.cloud.aws.credentials.access-key", localStack::getAccessKey);
+        registry.add("spring.cloud.aws.credentials.secret-key", localStack::getSecretKey);
+        registry.add("spring.cloud.aws.s3.endpoint", () -> localStack.getEndpointOverride(S3).toString());
+    }
+
+    @BeforeAll
+    static void beforeAll() throws IOException, InterruptedException {
+        localStack.execInContainer("awslocal", "s3", "mb", "s3://test-bucket");
+        localStack.execInContainer("awslocal", "s3", "cp", "/tmp/tmp-test-repositories.json",
+            "s3://test-bucket/s3-test-repositories.json");
+    }
+
+    @Test
+    public void testLoadRepositoryConfigurations() {
+        assertNotNull(repositories);
+
+        assertEquals(4, repositories.getAllConfigs().size());
+
+        RepositoryConfig j10p = repositories.getConfig("JScholarship-S3");
+        assertNotNull(j10p);
+
+        RepositoryConfig pubMed = repositories.getConfig("PubMed Central-S3");
+        assertNotNull(pubMed);
+
+        assertEquals("JScholarship-S3", j10p.getRepositoryKey());
+        assertEquals("PubMed Central-S3", pubMed.getRepositoryKey());
+
+        assertNotNull(j10p.getTransportConfig());
+        assertNotNull(j10p.getTransportConfig().getProtocolBinding());
+        assertNotNull(j10p.getAssemblerConfig().getSpec());
+
+        assertNotNull(pubMed.getTransportConfig());
+        assertNotNull(pubMed.getTransportConfig().getProtocolBinding());
+        assertNotNull(pubMed.getAssemblerConfig().getSpec());
+    }
+
+}

pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/config/spring/RepositoriesFactoryBeanConfigTest.java

@@ -36,7 +36,7 @@ public class RepositoriesFactoryBeanConfigTest {
     private Repositories repositories;
 
     @Test
-    public void foo() throws Exception {
+    public void testLoadRepositoryConfigurations() {
         assertNotNull(repositories);
 
         assertEquals(4, repositories.getAllConfigs().size());

pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/service/AbstractListenerIT.java

@@ -39,6 +39,7 @@
     "aws.region=us-east-1",
     "aws.sqs.endpoint.override=",
     "spring.jms.listener.auto-startup=true",
+    "spring.cloud.aws.s3.enabled=false",
     "pass.deposit.queue.submission.name=" + AwsSqsTestConfig.QUEUE_NAME,
     "pass.deposit.jobs.disabled=true"
 })
@@ -47,7 +48,7 @@
 public abstract class AbstractListenerIT {
 
     private static final DockerImageName LOCALSTACK_IMG =
-        DockerImageName.parse("localstack/localstack:2.1.0");
+        DockerImageName.parse("localstack/localstack:3.1.0");
 
     @Container
     static final LocalStackContainer LOCALSTACK_CONTAINER =

pass-deposit-services/deposit-core/src/test/resources/full-test-s3-repositories.json

@@ -0,0 +1,133 @@
+{
+  "JScholarship-S3": {
+    "deposit-config": {
+      "processing": {
+        "beanName": "org.eclipse.pass.deposit.status.DefaultDepositStatusProcessor"
+      }
+    },
+    "assembler": {
+      "specification": "simple",
+      "beanName": "simpleAssembler",
+      "options": {
+        "archive": "ZIP",
+        "compression": "NONE",
+        "algorithms": [
+          "sha512",
+          "md5"
+        ]
+      }
+    },
+    "transport-config": {
+      "protocol-binding": {
+        "protocol": "filesystem",
+        "baseDir": "target/packages",
+        "createIfMissing": "true",
+        "overwrite": "true"
+      }
+    }
+  },
+  "PubMed Central-S3": {
+    "assembler": {
+      "specification": "nihms-native-2017-07",
+      "beanName": "nihmsAssembler",
+      "options": {
+        "archive": "TAR",
+        "compression": "GZIP",
+        "algorithms": [
+          "sha512",
+          "md5"
+        ],
+        "funder-mapping": {
+          "johnshopkins.edu:funder:300032": "ahqr",
+          "johnshopkins.edu:funder:300293": "cdc",
+          "johnshopkins.edu:funder:300859": "cdc",
+          "johnshopkins.edu:funder:301459": "va",
+          "johnshopkins.edu:funder:300453": "epa",
+          "johnshopkins.edu:funder:303444": "hhmi",
+          "johnshopkins.edu:funder:300484": "nih",
+          "johnshopkins.edu:funder:300865": "nih",
+          "johnshopkins.edu:funder:300869": "nih",
+          "johnshopkins.edu:funder:300866": "nih",
+          "johnshopkins.edu:funder:308302": "nih",
+          "johnshopkins.edu:funder:305950": "nih",
+          "johnshopkins.edu:funder:302727": "nih",
+          "johnshopkins.edu:funder:300863": "nih",
+          "johnshopkins.edu:funder:300874": "nih",
+          "johnshopkins.edu:funder:300861": "nih",
+          "johnshopkins.edu:funder:300842": "nih",
+          "johnshopkins.edu:funder:303587": "nih",
+          "johnshopkins.edu:funder:303586": "nih",
+          "johnshopkins.edu:funder:306099": "nih",
+          "johnshopkins.edu:funder:300858": "nih",
+          "johnshopkins.edu:funder:301479": "nih",
+          "johnshopkins.edu:funder:300870": "nih",
+          "johnshopkins.edu:funder:303589": "nih",
+          "johnshopkins.edu:funder:300860": "nih",
+          "johnshopkins.edu:funder:300852": "nih",
+          "johnshopkins.edu:funder:302822": "nih",
+          "johnshopkins.edu:funder:302592": "nih",
+          "johnshopkins.edu:funder:303585": "nih",
+          "johnshopkins.edu:funder:303574": "nih",
+          "johnshopkins.edu:funder:300867": "nih",
+          "johnshopkins.edu:funder:303580": "nih",
+          "johnshopkins.edu:funder:301978": "nih",
+          "johnshopkins.edu:funder:305204": "aspr",
+          "johnshopkins.edu:funder:303395": "fda"
+        }
+      }
+    },
+    "transport-config": {
+      "protocol-binding": {
+        "protocol": "filesystem",
+        "baseDir": "target/packages",
+        "createIfMissing": "true",
+        "overwrite": "true"
+      }
+    }
+  },
+  "BagIt": {
+    "assembler": {
+      "specification": "simple",
+      "beanName": "simpleAssembler",
+      "options": {
+        "archive": "ZIP",
+        "compression": "NONE",
+        "algorithms": [
+          "sha512",
+          "md5"
+        ],
+        "baginfo-template-resource": "/bag-info.hbm"
+      }
+    },
+    "transport-config": {
+      "protocol-binding": {
+        "protocol": "filesystem",
+        "baseDir": "target/packages",
+        "createIfMissing": "true",
+        "overwrite": "true"
+      }
+    }
+  },
+  "dash": {
+    "assembler": {
+      "specification": "simple",
+      "beanName": "simpleAssembler",
+      "options": {
+        "archive": "ZIP",
+        "compression": "NONE",
+        "algorithms": [
+          "sha512",
+          "md5"
+        ]
+      }
+    },
+    "transport-config": {
+      "protocol-binding": {
+        "protocol": "filesystem",
+        "baseDir": "target/packages",
+        "createIfMissing": "true",
+        "overwrite": "true"
+      }
+    }
+  }
+}

pass-deposit-services/deposit-core/src/test/resources/test-application.properties

@@ -15,6 +15,7 @@
 #
 spring.jms.listener.auto-startup=false
 pass.deposit.jobs.disabled=true
+spring.cloud.aws.s3.enabled=false
 
 pass.client.url=http://localhost:8080/
 pass.client.user=test

pass-deposit-services/pom.xml

@@ -496,6 +496,7 @@
                 <ignoredUnusedDeclaredDependency>org.slf4j:log4j-over-slf4j:</ignoredUnusedDeclaredDependency>
                 <!-- mockito-inline is needed to mock final classes -->
                 <ignoredUnusedDeclaredDependency>org.mockito:mockito-inline:</ignoredUnusedDeclaredDependency>
+                <ignoredUnusedDeclaredDependency>io.awspring.cloud:spring-cloud-aws-starter-s3:</ignoredUnusedDeclaredDependency>
               </ignoredUnusedDeclaredDependencies>
               <ignoredNonTestScopedDependencies>
                 <!-- junit-jupiter is a module containing the junit api jars used directly -->