diff --git a/README.md b/README.md index b82fca9ef..22eac24e8 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,42 @@ If you also would like to test download count via Azure Blob, follow these steps * `AZURE_LOGS_SAS_TOKEN` with the shared access token for the `insights-logs-storageread` container. * If you change the variables in a running workspace, run `scripts/generate-properties.sh` in the `server` directory to update the application properties. +### Amazon S3 Setup + +If you would like to test file storage via Amazon S3, follow these steps: + +* Login to the AWS Console and create an [S3 storage bucket](https://s3.console.aws.amazon.com/s3/home?refid=ft_card) +* Go to the bucket's `Permissions` tab. + * Disable the `Block all public access` setting. + * Add a `Cross-origin resource sharing (CORS)` configuration: + ```json + [ + { + "AllowedHeaders": [ + "*" + ], + "AllowedMethods": [ + "GET", + "HEAD" + ], + "AllowedOrigins": [ + "*" + ], + "ExposeHeaders": [] + } + ] + ``` +* Follow the steps for [programmatic access](https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys) to create your access key id and secret access key +* Configure the following environment variables on your server environment + * `AWS_ACCESS_KEY_ID` with your access key id + * `AWS_SECRET_ACCESS_KEY` with your secret access key + * `AWS_REGION` with your bucket region name + * `AWS_SERVICE_ENDPOINT` with the url of your S3 provider if not using AWS (for AWS do not set) + * `AWS_BUCKET` with your bucket name + * `AWS_PATH_STYLE_ACCESS` whether or not to use path style access, (defaults to `false`) + * Path-style access: `https://s3..amazonaws.com//` + * Virtual-style access: `https://.s3..amazonaws.com/` + ## License [Eclipse Public License 2.0](https://www.eclipse.org/legal/epl-2.0/) diff --git a/server/build.gradle b/server/build.gradle index 821ed8f57..ed00a9354 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -24,6 +24,7 @@ def versions = [ springdoc: '2.1.0', gcloud: '2.36.1', azure: '12.23.0', + aws: '2.29.29', junit: '5.9.2', testcontainers: '1.15.2', jackson: '2.15.2', @@ -92,6 +93,7 @@ dependencies { implementation "org.flywaydb:flyway-core:${versions.flyway}" implementation "com.google.cloud:google-cloud-storage:${versions.gcloud}" implementation "com.azure:azure-storage-blob:${versions.azure}" + implementation "software.amazon.awssdk:s3:${versions.aws}" implementation "org.springdoc:springdoc-openapi-starter-webmvc-ui:${versions.springdoc}" implementation "com.fasterxml.jackson.core:jackson-core:${versions.jackson}" implementation "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}" diff --git a/server/scripts/generate-properties.sh b/server/scripts/generate-properties.sh index 4de5051c9..7a337b812 100755 --- a/server/scripts/generate-properties.sh +++ b/server/scripts/generate-properties.sh @@ -77,3 +77,23 @@ then echo "ovsx.logs.azure.sas-token=$AZURE_LOGS_SAS_TOKEN" >> $OVSX_APP_PROFILE echo "Using Azure Logs Storage: $AZURE_LOGS_SERVICE_ENDPOINT" fi + +# Set the AWS Storage service access key id, secret access key, region and endpoint +if [ -n "$AWS_ACCESS_KEY_ID" ] && [ -n "$AWS_SECRET_ACCESS_KEY" ] && [ -n "$AWS_REGION" ] && [ -n "$AWS_BUCKET" ] +then + echo "ovsx.storage.aws.access-key-id=$AWS_ACCESS_KEY_ID" >> $OVSX_APP_PROFILE + echo "ovsx.storage.aws.secret-access-key=$AWS_SECRET_ACCESS_KEY" >> $OVSX_APP_PROFILE + echo "ovsx.storage.aws.region=$AWS_REGION" >> $OVSX_APP_PROFILE + echo "ovsx.storage.aws.bucket=$AWS_BUCKET" >> $OVSX_APP_PROFILE + if [ -n "$AWS_PATH_STYLE_ACCESS" ] + then + echo "ovsx.storage.aws.path-style-access=$AWS_PATH_STYLE_ACCESS" >> $OVSX_APP_PROFILE + fi + if [ -n "$AWS_SERVICE_ENDPOINT" ] + then + echo "ovsx.storage.aws.service-endpoint=$AWS_SERVICE_ENDPOINT" >> $OVSX_APP_PROFILE + echo "Using AWS S3 Storage: $AWS_SERVICE_ENDPOINT" + else + echo "Using AWS S3 Storage." + fi +fi diff --git a/server/src/main/java/org/eclipse/openvsx/entities/FileResource.java b/server/src/main/java/org/eclipse/openvsx/entities/FileResource.java index d5b55a0f2..583678478 100644 --- a/server/src/main/java/org/eclipse/openvsx/entities/FileResource.java +++ b/server/src/main/java/org/eclipse/openvsx/entities/FileResource.java @@ -37,6 +37,7 @@ public class FileResource implements Serializable { public static final String STORAGE_LOCAL = "local"; public static final String STORAGE_GOOGLE = "google-cloud"; public static final String STORAGE_AZURE = "azure-blob"; + public static final String STORAGE_AWS = "aws"; @Id @GeneratedValue(generator = "fileResourceSeq") @@ -105,4 +106,4 @@ public String getStorageType() { public void setStorageType(String storageType) { this.storageType = storageType; } -} \ No newline at end of file +} diff --git a/server/src/main/java/org/eclipse/openvsx/storage/AwsStorageService.java b/server/src/main/java/org/eclipse/openvsx/storage/AwsStorageService.java new file mode 100644 index 000000000..55a204e63 --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/storage/AwsStorageService.java @@ -0,0 +1,272 @@ +/******************************************************************************** + * Copyright (c) 2022 Marshall Walker and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + ********************************************************************************/ + +package org.eclipse.openvsx.storage; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.HashMap; +import java.util.List; + +import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.lang3.StringUtils; +import org.eclipse.openvsx.entities.FileResource; +import org.eclipse.openvsx.entities.Namespace; +import org.eclipse.openvsx.util.TempFile; +import org.eclipse.openvsx.util.UrlUtil; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.cache.annotation.Cacheable; +import org.springframework.data.util.Pair; +import org.springframework.stereotype.Component; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.awscore.defaultsmode.DefaultsMode; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.endpoints.S3EndpointParams; +import software.amazon.awssdk.services.s3.endpoints.S3EndpointProvider; +import software.amazon.awssdk.services.s3.model.*; +import software.amazon.awssdk.services.s3.presigner.S3Presigner; +import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest; + +import static org.eclipse.openvsx.cache.CacheService.CACHE_EXTENSION_FILES; +import static org.eclipse.openvsx.cache.CacheService.GENERATOR_FILES; + +@Component +public class AwsStorageService implements IStorageService { + + private final FileCacheDurationConfig fileCacheDurationConfig; + + @Value("${ovsx.storage.aws.access-key-id:}") + String accessKeyId; + + @Value("${ovsx.storage.aws.secret-access-key:}") + String secretAccessKey; + + @Value("${ovsx.storage.aws.region:}") + String region; + + @Value("${ovsx.storage.aws.service-endpoint:}") + String serviceEndpoint; + + @Value("${ovsx.storage.aws.bucket:}") + String bucket; + + @Value("${ovsx.storage.aws.path-style-access:false}") + boolean pathStyleAccess; + + private S3Client s3Client; + + public AwsStorageService(FileCacheDurationConfig fileCacheDurationConfig) { + this.fileCacheDurationConfig = fileCacheDurationConfig; + } + + protected S3Client getS3Client() { + if (s3Client == null) { + var credentials = AwsBasicCredentials.create(accessKeyId, secretAccessKey); + var s3ClientBuilder = S3Client.builder() + .defaultsMode(DefaultsMode.STANDARD) + .forcePathStyle(pathStyleAccess) + .credentialsProvider(StaticCredentialsProvider.create(credentials)) + .region(Region.of(region)); + + if(StringUtils.isNotEmpty(serviceEndpoint)) { + var endpointParams = S3EndpointParams.builder() + .endpoint(serviceEndpoint) + .region(Region.of(region)) + .build(); + + var endpoint = S3EndpointProvider + .defaultProvider() + .resolveEndpoint(endpointParams).join(); + + s3ClientBuilder = s3ClientBuilder.endpointOverride(endpoint.url()); + } + + s3Client = s3ClientBuilder.build(); + } + return s3Client; + } + + private S3Presigner getS3Presigner() { + var credentials = AwsBasicCredentials.create(accessKeyId, secretAccessKey); + var builder = S3Presigner.builder() + .credentialsProvider(StaticCredentialsProvider.create(credentials)) + .region(Region.of(region)); + + if(StringUtils.isNotEmpty(serviceEndpoint)) { + var endpointParams = S3EndpointParams.builder() + .endpoint(serviceEndpoint) + .region(Region.of(region)) + .build(); + + var endpoint = S3EndpointProvider + .defaultProvider() + .resolveEndpoint(endpointParams).join(); + + builder = builder.endpointOverride(endpoint.url()); + } + + return builder.build(); + } + + @Override + public boolean isEnabled() { + return !StringUtils.isEmpty(accessKeyId); + } + + @Override + public void uploadFile(TempFile tempFile) { + var resource = tempFile.getResource(); + uploadFile(tempFile, resource.getName(), getObjectKey(resource)); + } + + @Override + public void uploadNamespaceLogo(TempFile logoFile) { + var namespace = logoFile.getNamespace(); + uploadFile(logoFile, namespace.getLogoName(), getObjectKey(namespace)); + } + + protected void uploadFile(TempFile file, String fileName, String objectKey) { + var metadata = new HashMap(); + metadata.put("Content-Type", StorageUtil.getFileType(fileName).toString()); + if (fileName.endsWith(".vsix")) { + metadata.put("Content-Disposition", "attachment; filename=\"" + fileName + "\""); + } else { + metadata.put("Cache-Control", StorageUtil.getCacheControl(fileName).getHeaderValue()); + } + + var request = PutObjectRequest.builder() + .bucket(bucket) + .key(objectKey) + .metadata(metadata) + .build(); + + getS3Client().putObject(request, file.getPath()); + } + + @Override + public void removeFile(FileResource resource) { + removeFile(getObjectKey(resource)); + } + + @Override + public void removeNamespaceLogo(Namespace namespace) { + removeFile(getObjectKey(namespace)); + } + + private void removeFile(String objectKey) { + var request = DeleteObjectRequest.builder() + .bucket(bucket) + .key(objectKey) + .build(); + + getS3Client().deleteObject(request); + } + + @Override + public URI getLocation(FileResource resource) { + return getLocation(getObjectKey(resource)); + } + + @Override + public URI getNamespaceLogoLocation(Namespace namespace) { + return getLocation(getObjectKey(namespace)); + } + + private URI getLocation(String objectKey) { + var objectRequest = GetObjectRequest.builder() + .bucket(bucket) + .key(objectKey) + .build(); + + var presignRequest = GetObjectPresignRequest.builder() + .signatureDuration(fileCacheDurationConfig.getCacheDuration()) + .getObjectRequest(objectRequest) + .build(); + + try (var presigner = getS3Presigner()) { + var presignedRequest = presigner.presignGetObject(presignRequest); + return presignedRequest.url().toURI(); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + } + + @Override + public TempFile downloadFile(FileResource resource) throws IOException { + var objectKey = getObjectKey(resource); + var request = GetObjectRequest.builder() + .bucket(bucket) + .key(objectKey) + .build(); + + var tempFile = new TempFile("temp_file_", ""); + try (var stream = getS3Client().getObject(request)) { + Files.copy(stream, tempFile.getPath(), StandardCopyOption.REPLACE_EXISTING); + } + tempFile.setResource(resource); + return tempFile; + } + + @Override + public void copyFiles(List> pairs) { + for(var pair : pairs) { + var oldObjectKey = getObjectKey(pair.getFirst()); + var newObjectKey = getObjectKey(pair.getSecond()); + var request = CopyObjectRequest.builder() + .sourceBucket(bucket) + .sourceKey(oldObjectKey) + .destinationBucket(bucket) + .destinationKey(newObjectKey) + .build(); + + getS3Client().copyObject(request); + } + } + + @Override + @Cacheable(value = CACHE_EXTENSION_FILES, keyGenerator = GENERATOR_FILES) + public Path getCachedFile(FileResource resource) throws IOException { + var objectKey = getObjectKey(resource); + var request = GetObjectRequest.builder() + .bucket(bucket) + .key(objectKey) + .build(); + + var path = Files.createTempFile("cached_file", null); + try (var stream = getS3Client().getObject(request)) { + Files.copy(stream, path, StandardCopyOption.REPLACE_EXISTING); + } + return path; + } + + protected String getObjectKey(FileResource resource) { + var extVersion = resource.getExtension(); + var extension = extVersion.getExtension(); + var namespace = extension.getNamespace(); + var segments = new String[] {namespace.getName(), extension.getName()}; + if (!extVersion.isUniversalTargetPlatform()) { + segments = ArrayUtils.add(segments, extVersion.getTargetPlatform()); + } + + segments = ArrayUtils.add(segments, extVersion.getVersion()); + segments = ArrayUtils.addAll(segments, resource.getName().split("/")); + return UrlUtil.createApiUrl("", segments).substring(1); // remove first '/' + } + + protected String getObjectKey(Namespace namespace) { + return UrlUtil.createApiUrl("", namespace.getName(), "logo", namespace.getLogoName()).substring(1); + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/storage/AzureBlobStorageService.java b/server/src/main/java/org/eclipse/openvsx/storage/AzureBlobStorageService.java index 10e07acef..ba8bcaa7d 100644 --- a/server/src/main/java/org/eclipse/openvsx/storage/AzureBlobStorageService.java +++ b/server/src/main/java/org/eclipse/openvsx/storage/AzureBlobStorageService.java @@ -141,7 +141,7 @@ private void removeFile(String blobName) { } } - @Override + @Override public URI getLocation(FileResource resource) { var blobName = getBlobName(resource); if (StringUtils.isEmpty(serviceEndpoint)) { diff --git a/server/src/main/java/org/eclipse/openvsx/storage/FileCacheDurationConfig.java b/server/src/main/java/org/eclipse/openvsx/storage/FileCacheDurationConfig.java new file mode 100644 index 000000000..39b26ed50 --- /dev/null +++ b/server/src/main/java/org/eclipse/openvsx/storage/FileCacheDurationConfig.java @@ -0,0 +1,26 @@ +/******************************************************************************** + * Copyright (c) 2022 Marshall Walker and others + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v. 2.0 which is available at + * http://www.eclipse.org/legal/epl-2.0. + * + * SPDX-License-Identifier: EPL-2.0 + ********************************************************************************/ + +package org.eclipse.openvsx.storage; + +import java.time.Duration; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Component +public class FileCacheDurationConfig { + + @Value("#{T(java.time.Duration).parse('${ovsx.storage.file-cache-duration:P7D}')}") + Duration cacheDuration; + + public Duration getCacheDuration() { + return cacheDuration; + } +} diff --git a/server/src/main/java/org/eclipse/openvsx/storage/GoogleCloudStorageService.java b/server/src/main/java/org/eclipse/openvsx/storage/GoogleCloudStorageService.java index d5ec61ec0..53fb09652 100644 --- a/server/src/main/java/org/eclipse/openvsx/storage/GoogleCloudStorageService.java +++ b/server/src/main/java/org/eclipse/openvsx/storage/GoogleCloudStorageService.java @@ -212,4 +212,4 @@ public Path getCachedFile(FileResource resource) throws IOException { getStorage().downloadTo(BlobId.of(bucketId, objectId), path); return path; } -} \ No newline at end of file +} diff --git a/server/src/main/java/org/eclipse/openvsx/storage/StorageMigration.java b/server/src/main/java/org/eclipse/openvsx/storage/StorageMigration.java index bd2c76ecc..da579f8d1 100644 --- a/server/src/main/java/org/eclipse/openvsx/storage/StorageMigration.java +++ b/server/src/main/java/org/eclipse/openvsx/storage/StorageMigration.java @@ -68,7 +68,7 @@ public void findResources(ApplicationStartedEvent event) { return; } - var migrations = new ArrayList<>(List.of(STORAGE_LOCAL, STORAGE_GOOGLE, STORAGE_AZURE)); + var migrations = new ArrayList<>(List.of(STORAGE_LOCAL, STORAGE_GOOGLE, STORAGE_AZURE, STORAGE_AWS)); migrations.remove(storageType); var migrationCount = new int[migrations.size()]; for (var i = 0; i < migrations.size(); i++) { diff --git a/server/src/main/java/org/eclipse/openvsx/storage/StorageUtilService.java b/server/src/main/java/org/eclipse/openvsx/storage/StorageUtilService.java index 815953689..8fea5cd82 100644 --- a/server/src/main/java/org/eclipse/openvsx/storage/StorageUtilService.java +++ b/server/src/main/java/org/eclipse/openvsx/storage/StorageUtilService.java @@ -51,10 +51,12 @@ public class StorageUtilService implements IStorageService { private final GoogleCloudStorageService googleStorage; private final AzureBlobStorageService azureStorage; private final LocalStorageService localStorage; + private final AwsStorageService awsStorage; private final AzureDownloadCountService azureDownloadCountService; private final SearchUtilService search; private final CacheService cache; private final EntityManager entityManager; + private final FileCacheDurationConfig fileCacheDurationConfig; /** Determines which external storage service to use in case multiple services are configured. */ @Value("${ovsx.storage.primary-service:}") @@ -69,19 +71,23 @@ public StorageUtilService( GoogleCloudStorageService googleStorage, AzureBlobStorageService azureStorage, LocalStorageService localStorage, + AwsStorageService awsStorage, AzureDownloadCountService azureDownloadCountService, SearchUtilService search, CacheService cache, - EntityManager entityManager + EntityManager entityManager, + FileCacheDurationConfig fileCacheDurationConfig ) { this.repositories = repositories; this.googleStorage = googleStorage; this.azureStorage = azureStorage; this.localStorage = localStorage; + this.awsStorage = awsStorage; this.azureDownloadCountService = azureDownloadCountService; this.search = search; this.cache = cache; this.entityManager = entityManager; + this.fileCacheDurationConfig = fileCacheDurationConfig; } public boolean shouldStoreExternally(FileResource resource) { @@ -96,15 +102,17 @@ private boolean shouldStoreLogoExternally() { @Override public boolean isEnabled() { - return googleStorage.isEnabled() || azureStorage.isEnabled() || localStorage.isEnabled(); + return googleStorage.isEnabled() || azureStorage.isEnabled() || localStorage.isEnabled() || awsStorage.isEnabled(); } public String getActiveStorageType() { - var storageTypes = new ArrayList(2); + var storageTypes = new ArrayList(3); if (googleStorage.isEnabled()) storageTypes.add(STORAGE_GOOGLE); if (azureStorage.isEnabled()) storageTypes.add(STORAGE_AZURE); + if (awsStorage.isEnabled()) + storageTypes.add(STORAGE_AWS); if (!StringUtils.isEmpty(primaryService)) { if (!storageTypes.contains(primaryService)) throw new RuntimeException("The selected primary storage service is not available."); @@ -131,6 +139,9 @@ public void uploadFile(TempFile tempFile) { case STORAGE_AZURE: azureStorage.uploadFile(tempFile); break; + case STORAGE_AWS: + awsStorage.uploadFile(tempFile); + break; case STORAGE_LOCAL: localStorage.uploadFile(tempFile); break; @@ -157,6 +168,9 @@ public void uploadNamespaceLogo(TempFile logoFile) { case STORAGE_AZURE: azureStorage.uploadNamespaceLogo(logoFile); break; + case STORAGE_AWS: + awsStorage.uploadNamespaceLogo(logoFile); + break; case STORAGE_LOCAL: localStorage.uploadNamespaceLogo(logoFile); break; @@ -176,6 +190,9 @@ public void removeFile(FileResource resource) { case STORAGE_AZURE: azureStorage.removeFile(resource); break; + case STORAGE_AWS: + awsStorage.removeFile(resource); + break; case STORAGE_LOCAL: localStorage.removeFile(resource); break; @@ -191,6 +208,9 @@ public void removeNamespaceLogo(Namespace namespace) { case STORAGE_AZURE: azureStorage.removeNamespaceLogo(namespace); break; + case STORAGE_AWS: + awsStorage.removeNamespaceLogo(namespace); + break; case STORAGE_LOCAL: localStorage.removeNamespaceLogo(namespace); break; @@ -202,6 +222,7 @@ public URI getLocation(FileResource resource) { return switch (resource.getStorageType()) { case STORAGE_GOOGLE -> googleStorage.getLocation(resource); case STORAGE_AZURE -> azureStorage.getLocation(resource); + case STORAGE_AWS -> awsStorage.getLocation(resource); case STORAGE_LOCAL -> localStorage.getLocation(resource); default -> null; }; @@ -212,6 +233,7 @@ public URI getNamespaceLogoLocation(Namespace namespace) { return switch (namespace.getLogoStorageType()) { case STORAGE_GOOGLE -> googleStorage.getNamespaceLogoLocation(namespace); case STORAGE_AZURE -> azureStorage.getNamespaceLogoLocation(namespace); + case STORAGE_AWS -> awsStorage.getNamespaceLogoLocation(namespace); case STORAGE_LOCAL -> localStorage.getNamespaceLogoLocation(namespace); default -> null; }; @@ -221,6 +243,7 @@ public TempFile downloadFile(FileResource resource) throws IOException { return switch (resource.getStorageType()) { case STORAGE_GOOGLE -> googleStorage.downloadFile(resource); case STORAGE_AZURE -> azureStorage.downloadFile(resource); + case STORAGE_AWS -> awsStorage.downloadFile(resource); case STORAGE_LOCAL -> localStorage.downloadFile(resource); default -> null; }; @@ -267,7 +290,7 @@ public ResponseEntity getFileResponse(FileResource resour } else { return ResponseEntity.status(HttpStatus.FOUND) .location(getLocation(resource)) - .cacheControl(CacheControl.maxAge(7, TimeUnit.DAYS).cachePublic()) + .cacheControl(CacheControl.maxAge(fileCacheDurationConfig.getCacheDuration()).cachePublic()) .build(); } } @@ -309,6 +332,9 @@ public void copyFiles(List> pairs) { case STORAGE_AZURE: azureStorage.copyFiles(group); break; + case STORAGE_AWS: + awsStorage.copyFiles(group); + break; case STORAGE_LOCAL: localStorage.copyFiles(group); break; @@ -321,6 +347,7 @@ public Path getCachedFile(FileResource resource) throws IOException { return switch (resource.getStorageType()) { case STORAGE_GOOGLE -> googleStorage.getCachedFile(resource); case STORAGE_AZURE -> azureStorage.getCachedFile(resource); + case STORAGE_AWS -> awsStorage.getCachedFile(resource); case STORAGE_LOCAL -> localStorage.getCachedFile(resource); default -> null; }; diff --git a/server/src/test/java/org/eclipse/openvsx/RegistryAPITest.java b/server/src/test/java/org/eclipse/openvsx/RegistryAPITest.java index aa32ff073..d39abcdbd 100644 --- a/server/src/test/java/org/eclipse/openvsx/RegistryAPITest.java +++ b/server/src/test/java/org/eclipse/openvsx/RegistryAPITest.java @@ -87,9 +87,9 @@ @AutoConfigureWebClient @MockBean({ ClientRegistrationRepository.class, UpstreamRegistryService.class, GoogleCloudStorageService.class, - AzureBlobStorageService.class, VSCodeIdService.class, AzureDownloadCountService.class, CacheService.class, - EclipseService.class, PublishExtensionVersionService.class, SimpleMeterRegistry.class, JobRequestScheduler.class, - ExtensionControlService.class + AzureBlobStorageService.class, AwsStorageService.class, VSCodeIdService.class, AzureDownloadCountService.class, + CacheService.class, EclipseService.class, PublishExtensionVersionService.class, SimpleMeterRegistry.class, + JobRequestScheduler.class, ExtensionControlService.class, FileCacheDurationConfig.class }) class RegistryAPITest { @@ -2425,6 +2425,7 @@ LocalRegistryService localRegistryService( StorageUtilService storageUtil, EclipseService eclipse, CacheService cache, + FileCacheDurationConfig fileCacheDurationConfig, ExtensionVersionIntegrityService integrityService ) { return new LocalRegistryService( @@ -2463,20 +2464,24 @@ StorageUtilService storageUtilService( GoogleCloudStorageService googleStorage, AzureBlobStorageService azureStorage, LocalStorageService localStorage, + AwsStorageService awsStorage, AzureDownloadCountService azureDownloadCountService, SearchUtilService search, CacheService cache, - EntityManager entityManager + EntityManager entityManager, + FileCacheDurationConfig fileCacheDurationConfig ) { return new StorageUtilService( repositories, googleStorage, azureStorage, localStorage, + awsStorage, azureDownloadCountService, search, cache, - entityManager + entityManager, + fileCacheDurationConfig ); } diff --git a/server/src/test/java/org/eclipse/openvsx/adapter/VSCodeAPITest.java b/server/src/test/java/org/eclipse/openvsx/adapter/VSCodeAPITest.java index bdd743504..24172cee1 100644 --- a/server/src/test/java/org/eclipse/openvsx/adapter/VSCodeAPITest.java +++ b/server/src/test/java/org/eclipse/openvsx/adapter/VSCodeAPITest.java @@ -56,7 +56,6 @@ import org.springframework.transaction.support.TransactionTemplate; import org.testcontainers.shaded.com.fasterxml.jackson.databind.ObjectMapper; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; @@ -66,9 +65,7 @@ import java.time.LocalDateTime; import java.util.*; import java.util.stream.Collectors; -import java.util.zip.ZipEntry; import java.util.zip.ZipFile; -import java.util.zip.ZipOutputStream; import static org.eclipse.openvsx.entities.FileResource.*; import static org.mockito.ArgumentMatchers.anyCollection; @@ -80,9 +77,9 @@ @AutoConfigureWebClient @MockBean({ ClientRegistrationRepository.class, GoogleCloudStorageService.class, AzureBlobStorageService.class, - AzureDownloadCountService.class, CacheService.class, UpstreamVSCodeService.class, - VSCodeIdService.class, EntityManager.class, EclipseService.class, ExtensionValidator.class, - SimpleMeterRegistry.class + AwsStorageService.class, AzureDownloadCountService.class, CacheService.class, UpstreamVSCodeService.class, + VSCodeIdService.class, EntityManager.class, EclipseService.class, ExtensionValidator.class, SimpleMeterRegistry.class, + FileCacheDurationConfig.class }) class VSCodeAPITest { @@ -929,20 +926,24 @@ StorageUtilService storageUtilService( GoogleCloudStorageService googleStorage, AzureBlobStorageService azureStorage, LocalStorageService localStorage, + AwsStorageService awsStorage, AzureDownloadCountService azureDownloadCountService, SearchUtilService search, CacheService cache, - EntityManager entityManager + EntityManager entityManager, + FileCacheDurationConfig fileCacheDurationConfig ) { return new StorageUtilService( repositories, googleStorage, azureStorage, localStorage, + awsStorage, azureDownloadCountService, search, cache, - entityManager + entityManager, + fileCacheDurationConfig ); } diff --git a/server/src/test/java/org/eclipse/openvsx/admin/AdminAPITest.java b/server/src/test/java/org/eclipse/openvsx/admin/AdminAPITest.java index 7a36e370a..b9a65926c 100644 --- a/server/src/test/java/org/eclipse/openvsx/admin/AdminAPITest.java +++ b/server/src/test/java/org/eclipse/openvsx/admin/AdminAPITest.java @@ -67,9 +67,9 @@ @AutoConfigureWebClient @MockBean({ ClientRegistrationRepository.class, UpstreamRegistryService.class, GoogleCloudStorageService.class, - AzureBlobStorageService.class, VSCodeIdService.class, AzureDownloadCountService.class, - CacheService.class, PublishExtensionVersionHandler.class, SearchUtilService.class, - EclipseService.class, SimpleMeterRegistry.class + AzureBlobStorageService.class, AwsStorageService.class, VSCodeIdService.class, AzureDownloadCountService.class, + CacheService.class, PublishExtensionVersionHandler.class, SearchUtilService.class, EclipseService.class, + SimpleMeterRegistry.class, FileCacheDurationConfig.class }) class AdminAPITest { @@ -1250,6 +1250,7 @@ LocalRegistryService localRegistryService( StorageUtilService storageUtil, EclipseService eclipse, CacheService cache, + FileCacheDurationConfig fileCacheDurationConfig, ExtensionVersionIntegrityService integrityService ) { return new LocalRegistryService( @@ -1288,20 +1289,24 @@ StorageUtilService storageUtilService( GoogleCloudStorageService googleStorage, AzureBlobStorageService azureStorage, LocalStorageService localStorage, + AwsStorageService awsStorage, AzureDownloadCountService azureDownloadCountService, SearchUtilService search, CacheService cache, - EntityManager entityManager + EntityManager entityManager, + FileCacheDurationConfig fileCacheDurationConfig ) { return new StorageUtilService( repositories, googleStorage, azureStorage, localStorage, + awsStorage, azureDownloadCountService, search, cache, - entityManager + entityManager, + fileCacheDurationConfig ); } diff --git a/server/src/test/java/org/eclipse/openvsx/eclipse/EclipseServiceTest.java b/server/src/test/java/org/eclipse/openvsx/eclipse/EclipseServiceTest.java index 6f0437f36..9c6b843cf 100644 --- a/server/src/test/java/org/eclipse/openvsx/eclipse/EclipseServiceTest.java +++ b/server/src/test/java/org/eclipse/openvsx/eclipse/EclipseServiceTest.java @@ -55,9 +55,8 @@ @ExtendWith(SpringExtension.class) @MockBean({ EntityManager.class, SearchUtilService.class, GoogleCloudStorageService.class, AzureBlobStorageService.class, - VSCodeIdService.class, AzureDownloadCountService.class, CacheService.class, - UserService.class, PublishExtensionVersionHandler.class, - SimpleMeterRegistry.class + AwsStorageService.class, VSCodeIdService.class, AzureDownloadCountService.class, CacheService.class, + UserService.class, PublishExtensionVersionHandler.class, SimpleMeterRegistry.class, FileCacheDurationConfig.class }) class EclipseServiceTest { @@ -292,20 +291,24 @@ StorageUtilService storageUtilService( GoogleCloudStorageService googleStorage, AzureBlobStorageService azureStorage, LocalStorageService localStorage, + AwsStorageService awsStorage, AzureDownloadCountService azureDownloadCountService, SearchUtilService search, CacheService cache, - EntityManager entityManager + EntityManager entityManager, + FileCacheDurationConfig fileCacheDurationConfig ) { return new StorageUtilService( repositories, googleStorage, azureStorage, localStorage, + awsStorage, azureDownloadCountService, search, cache, - entityManager + entityManager, + fileCacheDurationConfig ); }