WIP on version update for google cloud storage

CHANGELOG.md

@@ -22,8 +22,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `com.microsoft.azure:msal4j` from 1.17.0 to 1.17.1 ([#15945](https://github.com/opensearch-project/OpenSearch/pull/15945))
 - Bump `ch.qos.logback:logback-core` from 1.5.6 to 1.5.8 ([#15946](https://github.com/opensearch-project/OpenSearch/pull/15946))
 - Update protobuf from 3.25.4 to 3.25.5 ([#16011](https://github.com/opensearch-project/OpenSearch/pull/16011))
+- Bump `com.google.cloud:google-cloud-storage` from 1.113.1 to 2.43.0 ([#13677](https://github.com/opensearch-project/OpenSearch/pull/13677))
 
 ### Changed
+- Add support for docker compose v2 in TestFixturesPlugin ([#16049](https://github.com/opensearch-project/OpenSearch/pull/16049))
 
 
 ### Deprecated

buildSrc/src/main/java/org/opensearch/gradle/docker/DockerSupportService.java

@@ -106,6 +106,7 @@ public DockerAvailability getDockerAvailability() {
             Version version = null;
             boolean isVersionHighEnough = false;
             boolean isComposeAvailable = false;
+            boolean isComposeV2Available = false;
 
             // Check if the Docker binary exists
             final Optional<String> dockerBinary = getDockerPath();
@@ -129,6 +130,8 @@ public DockerAvailability getDockerAvailability() {
                         if (lastResult.isSuccess() && composePath.isPresent()) {
                             isComposeAvailable = runCommand(composePath.get(), "version").isSuccess();
                         }
+
+                        isComposeV2Available = runCommand(dockerPath, "compose", "version").isSuccess();
                     }
                 }
             }
@@ -138,6 +141,7 @@ public DockerAvailability getDockerAvailability() {
             this.dockerAvailability = new DockerAvailability(
                 isAvailable,
                 isComposeAvailable,
+                isComposeV2Available,
                 isVersionHighEnough,
                 dockerPath,
                 version,
@@ -356,6 +360,11 @@ public static class DockerAvailability {
          */
         public final boolean isComposeAvailable;
 
+        /**
+         * True if docker compose is available.
+         */
+        public final boolean isComposeV2Available;
+
         /**
          * True if the installed Docker version is &gt;= 17.05
          */
@@ -379,13 +388,15 @@ public static class DockerAvailability {
         DockerAvailability(
             boolean isAvailable,
             boolean isComposeAvailable,
+            boolean isComposeV2Available,
             boolean isVersionHighEnough,
             String path,
             Version version,
             Result lastCommand
         ) {
             this.isAvailable = isAvailable;
             this.isComposeAvailable = isComposeAvailable;
+            this.isComposeV2Available = isComposeV2Available;
             this.isVersionHighEnough = isVersionHighEnough;
             this.path = path;
             this.version = version;

buildSrc/src/main/java/org/opensearch/gradle/testfixtures/TestFixturesPlugin.java

@@ -171,7 +171,11 @@ public void execute(Task task) {
                 .findFirst();
 
             composeExtension.getExecutable().set(dockerCompose.isPresent() ? dockerCompose.get() : "/usr/bin/docker");
-            composeExtension.getUseDockerComposeV2().set(false);
+            if (dockerSupport.get().getDockerAvailability().isComposeV2Available) {
+                composeExtension.getUseDockerComposeV2().set(true);
+            } else if (dockerSupport.get().getDockerAvailability().isComposeAvailable) {
+                composeExtension.getUseDockerComposeV2().set(false);
+            }
 
             tasks.named("composeUp").configure(t -> {
                 // Avoid running docker-compose tasks in parallel in CI due to some issues on certain Linux distributions
@@ -228,7 +232,8 @@ private void maybeSkipTask(Provider<DockerSupportService> dockerSupport, TaskPro
 
     private void maybeSkipTask(Provider<DockerSupportService> dockerSupport, Task task) {
         task.onlyIf(spec -> {
-            boolean isComposeAvailable = dockerSupport.get().getDockerAvailability().isComposeAvailable;
+            boolean isComposeAvailable = dockerSupport.get().getDockerAvailability().isComposeV2Available
+                || dockerSupport.get().getDockerAvailability().isComposeAvailable;
             if (isComposeAvailable == false) {
                 LOGGER.info("Task {} requires docker-compose but it is unavailable. Task will be skipped.", task.getPath());
             }

plugins/repository-gcs/build.gradle

@@ -48,27 +48,29 @@ opensearchplugin {
 }
 
 versions << [
-  'google_auth': '1.7.0'
+  'google_auth': '1.27.0'
 ]
 
 dependencies {
-  api 'com.google.api:api-common:1.8.1'
-  api 'com.google.api:gax:2.35.0'
-  api 'com.google.api:gax-httpjson:2.42.0'
+  api 'com.google.api:api-common:2.31.0'
+  api 'com.google.api:gax:2.48.0'
+  api 'com.google.api:gax-httpjson:2.48.0'
 
-  api 'com.google.apis:google-api-services-storage:v1-rev20230617-2.0.0'
+  api 'com.google.apis:google-api-services-storage:v1-rev20240319-2.0.0'
 
-  api 'com.google.api-client:google-api-client:2.2.0'
+  api 'com.google.api-client:google-api-client:2.5.0'
 
-  api 'com.google.api.grpc:proto-google-common-protos:2.37.1'
-  api 'com.google.api.grpc:proto-google-iam-v1:1.33.0'
+  api 'com.google.api.grpc:proto-google-common-protos:2.39.0'
+  api 'com.google.api.grpc:proto-google-cloud-storage-v2:2.43.0-beta'
+  api 'com.google.api.grpc:proto-google-iam-v1:1.34.0'
 
   api "com.google.auth:google-auth-library-credentials:${versions.google_auth}"
   api "com.google.auth:google-auth-library-oauth2-http:${versions.google_auth}"
 
-  api 'com.google.cloud:google-cloud-core:2.30.0'
-  api 'com.google.cloud:google-cloud-core-http:2.23.0'
-  api 'com.google.cloud:google-cloud-storage:1.113.1'
+  api 'com.google.cloud:google-cloud-core:2.43.0'
+  api 'com.google.cloud:google-cloud-core-http:2.43.0'
+  api 'com.google.cloud:google-cloud-storage:2.43.0'
+  implementation 'org.checkerframework:checker-qual:3.29.0'
 
   api 'com.google.code.gson:gson:2.10.1'
 
@@ -80,7 +82,7 @@ dependencies {
   api "com.google.http-client:google-http-client-gson:${versions.google_http_client}"
   api "com.google.http-client:google-http-client-jackson2:${versions.google_http_client}"
 
-  api 'com.google.oauth-client:google-oauth-client:1.34.1'
+  api 'com.google.oauth-client:google-oauth-client:1.36.0'
 
   api "commons-logging:commons-logging:${versions.commonslogging}"
   api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
@@ -148,11 +150,54 @@ thirdPartyAudit {
     'com.google.appengine.api.urlfetch.HTTPResponse',
     'com.google.appengine.api.urlfetch.URLFetchService',
     'com.google.appengine.api.urlfetch.URLFetchServiceFactory',
-    'com.google.auth.oauth2.GdchCredentials',
+    'com.google.cloud.opentelemetry.metric.GoogleCloudMetricExporter',
+    'com.google.cloud.opentelemetry.metric.MetricConfiguration',
+    'com.google.cloud.opentelemetry.metric.MetricConfiguration$Builder',
     'com.google.protobuf.util.JsonFormat',
     'com.google.protobuf.util.JsonFormat$Parser',
     'com.google.protobuf.util.JsonFormat$Printer',
     'com.google.protobuf.util.Timestamps',
+    'com.google.storage.v2.StorageClient',
+    'com.google.storage.v2.StorageClient$ListBucketsPagedResponse',
+    'com.google.storage.v2.StorageClient$ListHmacKeysPagedResponse',
+    'com.google.storage.v2.StorageClient$ListNotificationConfigsPagedResponse',
+    'com.google.storage.v2.StorageSettings',
+    'com.google.storage.v2.StorageSettings$Builder',
+    'com.google.storage.v2.stub.GrpcStorageStub',
+    'com.google.storage.v2.stub.StorageStubSettings',
+
+    'io.grpc.opentelemetry.GrpcOpenTelemetry',
+    'io.grpc.opentelemetry.GrpcOpenTelemetry$Builder',
+    'io.opentelemetry.api.OpenTelemetry',
+    'io.opentelemetry.api.common.AttributeKey',
+    'io.opentelemetry.api.common.Attributes',
+    'io.opentelemetry.api.common.AttributesBuilder',
+    'io.opentelemetry.api.internal.StringUtils',
+    'io.opentelemetry.api.metrics.DoubleHistogram',
+    'io.opentelemetry.api.metrics.DoubleHistogramBuilder',
+    'io.opentelemetry.api.metrics.LongCounter',
+    'io.opentelemetry.api.metrics.LongCounterBuilder',
+    'io.opentelemetry.api.metrics.Meter',
+    'io.opentelemetry.api.metrics.MeterBuilder',
+    'io.opentelemetry.contrib.gcp.resource.GCPResourceProvider',
+    'io.opentelemetry.sdk.OpenTelemetrySdk',
+    'io.opentelemetry.sdk.OpenTelemetrySdkBuilder',
+    'io.opentelemetry.sdk.common.CompletableResultCode',
+    'io.opentelemetry.sdk.common.export.MemoryMode',
+    'io.opentelemetry.sdk.metrics.Aggregation',
+    'io.opentelemetry.sdk.metrics.InstrumentSelector',
+    'io.opentelemetry.sdk.metrics.InstrumentSelectorBuilder',
+    'io.opentelemetry.sdk.metrics.InstrumentType',
+    'io.opentelemetry.sdk.metrics.SdkMeterProvider',
+    'io.opentelemetry.sdk.metrics.SdkMeterProviderBuilder',
+    'io.opentelemetry.sdk.metrics.View',
+    'io.opentelemetry.sdk.metrics.ViewBuilder',
+    'io.opentelemetry.sdk.metrics.data.AggregationTemporality',
+    'io.opentelemetry.sdk.metrics.export.DefaultAggregationSelector',
+    'io.opentelemetry.sdk.metrics.export.MetricExporter',
+    'io.opentelemetry.sdk.metrics.export.PeriodicMetricReader',
+    'io.opentelemetry.sdk.metrics.export.PeriodicMetricReaderBuilder',
+    'io.opentelemetry.sdk.resources.Resource',
     // commons-logging optional dependencies
     'org.apache.avalon.framework.logger.Logger',
     'org.apache.log.Hierarchy',
@@ -207,9 +252,17 @@ thirdPartyAudit {
     'javax.jms.Message',
     'javax.servlet.ServletContextEvent',
     'javax.servlet.ServletContextListener',
-    // Bump for gax 2.42.0
-    'com.google.api.gax.rpc.EndpointContext',
-    'com.google.api.gax.rpc.RequestMutator'
+    'com.google.api.gax.grpc.GrpcCallContext',
+    'com.google.api.gax.grpc.GrpcCallSettings',
+    'com.google.api.gax.grpc.GrpcCallSettings$Builder',
+    'com.google.api.gax.grpc.GrpcInterceptorProvider',
+    'com.google.api.gax.grpc.GrpcStatusCode',
+    'com.google.api.gax.grpc.GrpcStubCallableFactory',
+    'com.google.api.gax.grpc.InstantiatingGrpcChannelProvider',
+    'com.google.api.gax.grpc.InstantiatingGrpcChannelProvider$Builder',
+    'com.google.cloud.grpc.GrpcTransportOptions',
+    'com.google.cloud.grpc.GrpcTransportOptions$Builder',
+    'io.grpc.protobuf.ProtoUtils'
   )
 }
 

plugins/repository-gcs/licenses/api-common-2.31.0.jar.sha1

@@ -0,0 +1 @@
+a559a225c748f2b1f488d0bacb8f28c5d6791295

plugins/repository-gcs/licenses/checker-qual-3.29.0.jar.sha1

@@ -0,0 +1 @@
+a805a761b7a734d69f9c1766ead1ff78698a1497