diff --git a/pass-deposit-services/deposit-core/pom.xml b/pass-deposit-services/deposit-core/pom.xml index dc1ea4bc..98d61baf 100644 --- a/pass-deposit-services/deposit-core/pom.xml +++ b/pass-deposit-services/deposit-core/pom.xml @@ -35,8 +35,21 @@ 2.0.1 1.1.4 3.9.6 + 3.1.0 + + + + io.awspring.cloud + spring-cloud-aws-dependencies + ${awsspring.version} + pom + import + + + + @@ -64,6 +77,11 @@ spring-integration-mail + + io.awspring.cloud + spring-cloud-aws-starter-s3 + + org.jsoup jsoup diff --git a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/config/spring/RepositoriesFactoryBeanConfig.java b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/config/spring/RepositoriesFactoryBeanConfig.java index 7d5c5930..39e1264f 100644 --- a/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/config/spring/RepositoriesFactoryBeanConfig.java +++ b/pass-deposit-services/deposit-core/src/main/java/org/eclipse/pass/deposit/config/spring/RepositoriesFactoryBeanConfig.java @@ -32,9 +32,6 @@ public class RepositoriesFactoryBeanConfig { private static final Logger LOG = LoggerFactory.getLogger(RepositoriesFactoryBeanConfig.class); - @Value("${pass.deposit.repository.configuration}") - private Resource repositoryConfigResource; - @Bean public ObjectMapper repositoriesMapper(Environment env) { ObjectMapper mapper = new ObjectMapper(); diff --git a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/config/spring/RepositoriesConfigS3IT.java b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/config/spring/RepositoriesConfigS3IT.java new file mode 100644 index 00000000..caa4e7dd --- /dev/null +++ b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/config/spring/RepositoriesConfigS3IT.java @@ -0,0 +1,107 @@ +/* + * Copyright 2024 Johns Hopkins University + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.eclipse.pass.deposit.config.spring; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.testcontainers.containers.localstack.LocalStackContainer.Service.S3; + +import java.io.IOException; + +import org.eclipse.pass.deposit.DepositApp; +import org.eclipse.pass.deposit.config.repository.Repositories; +import org.eclipse.pass.deposit.config.repository.RepositoryConfig; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit.jupiter.SpringExtension; +import org.testcontainers.containers.BindMode; +import org.testcontainers.containers.localstack.LocalStackContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.DockerImageName; + +/** + * @author Russ Poetker (rpoetke1@jh.edu) + */ +@ExtendWith(SpringExtension.class) +@SpringBootTest(classes = DepositApp.class) +@TestPropertySource(properties = { + "spring.jms.listener.auto-startup=false", + "spring.cloud.aws.s3.enabled=true", + "pass.deposit.repository.configuration=s3://test-bucket/s3-test-repositories.json" +}) +@Testcontainers +public class RepositoriesConfigS3IT { + + private static final DockerImageName LOCALSTACK_IMG = + DockerImageName.parse("localstack/localstack:3.1.0"); + + @Container + static final LocalStackContainer localStack = + new LocalStackContainer(LOCALSTACK_IMG) + .withClasspathResourceMapping("/full-test-s3-repositories.json", + "/tmp/tmp-test-repositories.json", BindMode.READ_ONLY) + .withServices(S3); + + @Autowired + private Repositories repositories; + + @DynamicPropertySource + static void overrideProperties(DynamicPropertyRegistry registry) { + registry.add("spring.cloud.aws.region.static", localStack::getRegion); + registry.add("spring.cloud.aws.credentials.access-key", localStack::getAccessKey); + registry.add("spring.cloud.aws.credentials.secret-key", localStack::getSecretKey); + registry.add("spring.cloud.aws.s3.endpoint", () -> localStack.getEndpointOverride(S3).toString()); + } + + @BeforeAll + static void beforeAll() throws IOException, InterruptedException { + localStack.execInContainer("awslocal", "s3", "mb", "s3://test-bucket"); + localStack.execInContainer("awslocal", "s3", "cp", "/tmp/tmp-test-repositories.json", + "s3://test-bucket/s3-test-repositories.json"); + } + + @Test + public void testLoadRepositoryConfigurations() { + assertNotNull(repositories); + + assertEquals(4, repositories.getAllConfigs().size()); + + RepositoryConfig j10p = repositories.getConfig("JScholarship-S3"); + assertNotNull(j10p); + + RepositoryConfig pubMed = repositories.getConfig("PubMed Central-S3"); + assertNotNull(pubMed); + + assertEquals("JScholarship-S3", j10p.getRepositoryKey()); + assertEquals("PubMed Central-S3", pubMed.getRepositoryKey()); + + assertNotNull(j10p.getTransportConfig()); + assertNotNull(j10p.getTransportConfig().getProtocolBinding()); + assertNotNull(j10p.getAssemblerConfig().getSpec()); + + assertNotNull(pubMed.getTransportConfig()); + assertNotNull(pubMed.getTransportConfig().getProtocolBinding()); + assertNotNull(pubMed.getAssemblerConfig().getSpec()); + } + +} diff --git a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/config/spring/RepositoriesFactoryBeanConfigTest.java b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/config/spring/RepositoriesFactoryBeanConfigTest.java index e03a6e40..00d24335 100644 --- a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/config/spring/RepositoriesFactoryBeanConfigTest.java +++ b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/config/spring/RepositoriesFactoryBeanConfigTest.java @@ -36,7 +36,7 @@ public class RepositoriesFactoryBeanConfigTest { private Repositories repositories; @Test - public void foo() throws Exception { + public void testLoadRepositoryConfigurations() { assertNotNull(repositories); assertEquals(4, repositories.getAllConfigs().size()); diff --git a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/service/AbstractListenerIT.java b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/service/AbstractListenerIT.java index 04e904a3..baeb0dd1 100644 --- a/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/service/AbstractListenerIT.java +++ b/pass-deposit-services/deposit-core/src/test/java/org/eclipse/pass/deposit/service/AbstractListenerIT.java @@ -39,6 +39,7 @@ "aws.region=us-east-1", "aws.sqs.endpoint.override=", "spring.jms.listener.auto-startup=true", + "spring.cloud.aws.s3.enabled=false", "pass.deposit.queue.submission.name=" + AwsSqsTestConfig.QUEUE_NAME, "pass.deposit.jobs.disabled=true" }) @@ -47,7 +48,7 @@ public abstract class AbstractListenerIT { private static final DockerImageName LOCALSTACK_IMG = - DockerImageName.parse("localstack/localstack:2.1.0"); + DockerImageName.parse("localstack/localstack:3.1.0"); @Container static final LocalStackContainer LOCALSTACK_CONTAINER = diff --git a/pass-deposit-services/deposit-core/src/test/resources/full-test-s3-repositories.json b/pass-deposit-services/deposit-core/src/test/resources/full-test-s3-repositories.json new file mode 100644 index 00000000..c27948ff --- /dev/null +++ b/pass-deposit-services/deposit-core/src/test/resources/full-test-s3-repositories.json @@ -0,0 +1,133 @@ +{ + "JScholarship-S3": { + "deposit-config": { + "processing": { + "beanName": "org.eclipse.pass.deposit.status.DefaultDepositStatusProcessor" + } + }, + "assembler": { + "specification": "simple", + "beanName": "simpleAssembler", + "options": { + "archive": "ZIP", + "compression": "NONE", + "algorithms": [ + "sha512", + "md5" + ] + } + }, + "transport-config": { + "protocol-binding": { + "protocol": "filesystem", + "baseDir": "target/packages", + "createIfMissing": "true", + "overwrite": "true" + } + } + }, + "PubMed Central-S3": { + "assembler": { + "specification": "nihms-native-2017-07", + "beanName": "nihmsAssembler", + "options": { + "archive": "TAR", + "compression": "GZIP", + "algorithms": [ + "sha512", + "md5" + ], + "funder-mapping": { + "johnshopkins.edu:funder:300032": "ahqr", + "johnshopkins.edu:funder:300293": "cdc", + "johnshopkins.edu:funder:300859": "cdc", + "johnshopkins.edu:funder:301459": "va", + "johnshopkins.edu:funder:300453": "epa", + "johnshopkins.edu:funder:303444": "hhmi", + "johnshopkins.edu:funder:300484": "nih", + "johnshopkins.edu:funder:300865": "nih", + "johnshopkins.edu:funder:300869": "nih", + "johnshopkins.edu:funder:300866": "nih", + "johnshopkins.edu:funder:308302": "nih", + "johnshopkins.edu:funder:305950": "nih", + "johnshopkins.edu:funder:302727": "nih", + "johnshopkins.edu:funder:300863": "nih", + "johnshopkins.edu:funder:300874": "nih", + "johnshopkins.edu:funder:300861": "nih", + "johnshopkins.edu:funder:300842": "nih", + "johnshopkins.edu:funder:303587": "nih", + "johnshopkins.edu:funder:303586": "nih", + "johnshopkins.edu:funder:306099": "nih", + "johnshopkins.edu:funder:300858": "nih", + "johnshopkins.edu:funder:301479": "nih", + "johnshopkins.edu:funder:300870": "nih", + "johnshopkins.edu:funder:303589": "nih", + "johnshopkins.edu:funder:300860": "nih", + "johnshopkins.edu:funder:300852": "nih", + "johnshopkins.edu:funder:302822": "nih", + "johnshopkins.edu:funder:302592": "nih", + "johnshopkins.edu:funder:303585": "nih", + "johnshopkins.edu:funder:303574": "nih", + "johnshopkins.edu:funder:300867": "nih", + "johnshopkins.edu:funder:303580": "nih", + "johnshopkins.edu:funder:301978": "nih", + "johnshopkins.edu:funder:305204": "aspr", + "johnshopkins.edu:funder:303395": "fda" + } + } + }, + "transport-config": { + "protocol-binding": { + "protocol": "filesystem", + "baseDir": "target/packages", + "createIfMissing": "true", + "overwrite": "true" + } + } + }, + "BagIt": { + "assembler": { + "specification": "simple", + "beanName": "simpleAssembler", + "options": { + "archive": "ZIP", + "compression": "NONE", + "algorithms": [ + "sha512", + "md5" + ], + "baginfo-template-resource": "/bag-info.hbm" + } + }, + "transport-config": { + "protocol-binding": { + "protocol": "filesystem", + "baseDir": "target/packages", + "createIfMissing": "true", + "overwrite": "true" + } + } + }, + "dash": { + "assembler": { + "specification": "simple", + "beanName": "simpleAssembler", + "options": { + "archive": "ZIP", + "compression": "NONE", + "algorithms": [ + "sha512", + "md5" + ] + } + }, + "transport-config": { + "protocol-binding": { + "protocol": "filesystem", + "baseDir": "target/packages", + "createIfMissing": "true", + "overwrite": "true" + } + } + } +} \ No newline at end of file diff --git a/pass-deposit-services/deposit-core/src/test/resources/test-application.properties b/pass-deposit-services/deposit-core/src/test/resources/test-application.properties index f6557152..fcffe763 100644 --- a/pass-deposit-services/deposit-core/src/test/resources/test-application.properties +++ b/pass-deposit-services/deposit-core/src/test/resources/test-application.properties @@ -15,6 +15,7 @@ # spring.jms.listener.auto-startup=false pass.deposit.jobs.disabled=true +spring.cloud.aws.s3.enabled=false pass.client.url=http://localhost:8080/ pass.client.user=test diff --git a/pass-deposit-services/pom.xml b/pass-deposit-services/pom.xml index 55d615cf..dc440e63 100644 --- a/pass-deposit-services/pom.xml +++ b/pass-deposit-services/pom.xml @@ -496,6 +496,7 @@ org.slf4j:log4j-over-slf4j: org.mockito:mockito-inline: + io.awspring.cloud:spring-cloud-aws-starter-s3: