Merge branch 'main' into ipfield_iodvq

Signed-off-by: Harsha Vamsi Kalluri <[email protected]>

.github/workflows/version.yml

@@ -1,28 +1,32 @@
 name: Increment Version
 
 on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: 'the tag'
+        required: true
+        type: string
   push:
     tags:
       - '*.*.*'
 
-permissions: {}
+permissions:
+  contents: write
+  issues: write
+  pull-requests: write
+
 jobs:
   build:
     if: github.repository == 'opensearch-project/OpenSearch'
     runs-on: ubuntu-latest
     steps:
-      - name: GitHub App token
-        id: github_app_token
-        uses: tibdex/[email protected]
-        with:
-          app_id: ${{ secrets.APP_ID }}
-          private_key: ${{ secrets.APP_PRIVATE_KEY }}
-          installation_id: 22958780
-
-      - uses: actions/checkout@v4
-      - name: Fetch Tag and Version Information
+      - name: Fetch tag and version information
         run: |
           TAG=$(echo "${GITHUB_REF#refs/*/}")
+          if [ -n ${{ github.event.inputs.tag }} ]; then
+            TAG=${{ github.event.inputs.tag }}
+          fi
           CURRENT_VERSION_ARRAY=($(echo "$TAG" | tr . '\n'))
           BASE=$(IFS=. ; echo "${CURRENT_VERSION_ARRAY[*]:0:2}")
           BASE_X=$(IFS=. ; echo "${CURRENT_VERSION_ARRAY[*]:0:1}.x")
@@ -44,24 +48,22 @@ jobs:
           echo "NEXT_VERSION=$NEXT_VERSION" >> $GITHUB_ENV
           echo "NEXT_VERSION_UNDERSCORE=$NEXT_VERSION_UNDERSCORE" >> $GITHUB_ENV
           echo "NEXT_VERSION_ID=$NEXT_VERSION_ID" >> $GITHUB_ENV
+
       - uses: actions/checkout@v4
         with:
           ref: ${{ env.BASE }}
-          token: ${{ steps.github_app_token.outputs.token }}
 
-      - name: Increment Patch Version
-        run: |
-          echo Incrementing $CURRENT_VERSION to $NEXT_VERSION
-          echo "  - \"$CURRENT_VERSION\"" >> .ci/bwcVersions
-          sed -i "s/opensearch        = $CURRENT_VERSION/opensearch        = $NEXT_VERSION/g" buildSrc/version.properties
-          echo Adding $NEXT_VERSION_UNDERSCORE after $CURRENT_VERSION_UNDERSCORE
-          sed -i "s/public static final Version $CURRENT_VERSION_UNDERSCORE = new Version(\([[:digit:]]\+\)\(.*\));/\0\n    public static final Version $NEXT_VERSION_UNDERSCORE = new Version($NEXT_VERSION_ID\2);/g" libs/core/src/main/java/org/opensearch/Version.java
-          sed -i "s/CURRENT = $CURRENT_VERSION_UNDERSCORE;/CURRENT = $NEXT_VERSION_UNDERSCORE;/g" libs/core/src/main/java/org/opensearch/Version.java
+      - name: Increment Patch Version on Major.Minor branch
+        uses: peternied/opensearch-core-version-updater@v1
+        with:
+          previous-version: ${{ env.CURRENT_VERSION }}
+          new-version: ${{ env.NEXT_VERSION }}
+          update-current: true
 
-      - name: Create Pull Request
+      - name: Create PR for BASE
+        id: base_pr
         uses: peter-evans/create-pull-request@v5
         with:
-          token: ${{ steps.github_app_token.outputs.token }}
           base: ${{ env.BASE }}
           branch: 'create-pull-request/patch-${{ env.BASE }}'
           commit-message: Increment version to ${{ env.NEXT_VERSION }}
@@ -76,19 +78,18 @@ jobs:
       - uses: actions/checkout@v4
         with:
           ref: ${{ env.BASE_X }}
-          token: ${{ steps.github_app_token.outputs.token }}
 
-      - name: Add bwc version to .X branch
-        run: |
-          echo Adding bwc version $NEXT_VERSION after $CURRENT_VERSION
-          sed -i "s/- \"$CURRENT_VERSION\"/\0\n  - \"$NEXT_VERSION\"/g" .ci/bwcVersions
-          echo Adding $NEXT_VERSION_UNDERSCORE after $CURRENT_VERSION_UNDERSCORE
-          sed -i "s/public static final Version $CURRENT_VERSION_UNDERSCORE = new Version(\([[:digit:]]\+\)\(.*\));/\0\n    public static final Version $NEXT_VERSION_UNDERSCORE = new Version($NEXT_VERSION_ID\2);/g" libs/core/src/main/java/org/opensearch/Version.java
+      - name: Add Patch Version on Major.X branch
+        uses: peternied/opensearch-core-version-updater@v1
+        with:
+          previous-version: ${{ env.CURRENT_VERSION }}
+          new-version: ${{ env.NEXT_VERSION }}
+          update-current: false
 
-      - name: Create Pull Request
+      - name: Create PR for BASE_X
+        id: base_x_pr
         uses: peter-evans/create-pull-request@v5
         with:
-          token: ${{ steps.github_app_token.outputs.token }}
           base: ${{ env.BASE_X }}
           branch: 'create-pull-request/patch-${{ env.BASE_X }}'
           commit-message: Add bwc version ${{ env.NEXT_VERSION }}
@@ -103,19 +104,18 @@ jobs:
       - uses: actions/checkout@v4
         with:
           ref: main
-          token: ${{ steps.github_app_token.outputs.token }}
 
-      - name: Add bwc version to main branch
-        run: |
-          echo Adding bwc version $NEXT_VERSION after $CURRENT_VERSION
-          sed -i "s/- \"$CURRENT_VERSION\"/\0\n  - \"$NEXT_VERSION\"/g" .ci/bwcVersions
-          echo Adding $NEXT_VERSION_UNDERSCORE after $CURRENT_VERSION_UNDERSCORE
-          sed -i "s/public static final Version $CURRENT_VERSION_UNDERSCORE = new Version(\([[:digit:]]\+\)\(.*\));/\0\n    public static final Version $NEXT_VERSION_UNDERSCORE = new Version($NEXT_VERSION_ID\2);/g" libs/core/src/main/java/org/opensearch/Version.java
+      - name: Add Patch Version on main branch
+        uses: peternied/opensearch-core-version-updater@v1
+        with:
+          previous-version: ${{ env.CURRENT_VERSION }}
+          new-version: ${{ env.NEXT_VERSION }}
+          update-current: false
 
-      - name: Create Pull Request
+      - name: Create PR for main
+        id: main_pr
         uses: peter-evans/create-pull-request@v5
         with:
-          token: ${{ steps.github_app_token.outputs.token }}
           base: main
           branch: 'create-pull-request/patch-main'
           commit-message: Add bwc version ${{ env.NEXT_VERSION }}
@@ -126,3 +126,32 @@ jobs:
           title: '[AUTO] [main] Add bwc version ${{ env.NEXT_VERSION }}.'
           body: |
             I've noticed that a new tag ${{ env.TAG }} was pushed, and added a bwc version ${{ env.NEXT_VERSION }}.
+
+      - name: Create tracking issue
+        id: create-issue
+        uses: actions/[email protected]
+        with:
+          script: |
+            const body = `
+             ### Description
+             A new version of OpenSearch was released, to prepare for the next release new version numbers need to be updated in all active branches of development.
+
+             ### Exit Criteria
+             Review and merged the following pull requests
+             - [ ] ${{ steps.base_pr.outputs.pull-request-url }}
+             - [ ] ${{ steps.base_x_pr.outputs.pull-request-url }}
+             - [ ] ${{ steps.main_pr.outputs.pull-request-url }}
+
+             ### Additional Context
+             See project wide guidance on branching and versions [[link]](https://github.com/opensearch-project/.github/blob/main/RELEASING.md).
+            `
+            const { data: issue }= await github.rest.issues.create({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              labels: ["Build"],
+              title: "Increment version for ${{ env.NEXT_VERSION }}",
+              body: body
+            });
+            console.error(JSON.stringify(issue));
+            return issue.number;
+          result-encoding: string

CHANGELOG.md

@@ -174,8 +174,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `net.java.dev.jna:jna` from 5.13.0 to 5.14.0 ([#11798](https://github.com/opensearch-project/OpenSearch/pull/11798))
 - Bump `lycheeverse/lychee-action` from 1.8.0 to 1.9.1 ([#11795](https://github.com/opensearch-project/OpenSearch/pull/11795), [#11887](https://github.com/opensearch-project/OpenSearch/pull/11887))
 - Bump `Lucene` from 9.8.0 to 9.9.1 ([#11421](https://github.com/opensearch-project/OpenSearch/pull/11421))
-- Bump `com.networknt:json-schema-validator` from 1.0.86 to 1.1.0 ([#11886](https://github.com/opensearch-project/OpenSearch/pull/11886))
+- Bump `com.networknt:json-schema-validator` from 1.0.86 to 1.2.0 ([#11886](https://github.com/opensearch-project/OpenSearch/pull/11886), [#11963](https://github.com/opensearch-project/OpenSearch/pull/11963))
 - Bump `com.google.api:gax-httpjson` from 0.103.1 to 2.39.0 ([#11794](https://github.com/opensearch-project/OpenSearch/pull/11794))
+- Bump `com.google.oauth-client:google-oauth-client` from 1.34.1 to 1.35.0 ([#11960](https://github.com/opensearch-project/OpenSearch/pull/11960))
+- Bump `com.diffplug.spotless` from 6.23.2 to 6.24.0 ([#11962](https://github.com/opensearch-project/OpenSearch/pull/11962))
+- Bump `com.google.cloud:google-cloud-core` from 2.5.10 to 2.30.0 ([#11961](https://github.com/opensearch-project/OpenSearch/pull/11961))
 
 ### Changed
 - Mute the query profile IT with concurrent execution ([#9840](https://github.com/opensearch-project/OpenSearch/pull/9840))
@@ -208,6 +211,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Ensure Jackson default maximums introduced in 2.16.0 do not conflict with OpenSearch settings ([#11890](https://github.com/opensearch-project/OpenSearch/pull/11890))
 - Extract cluster management for integration tests into JUnit test rule out of OpenSearchIntegTestCase ([#11877](https://github.com/opensearch-project/OpenSearch/pull/11877))
 - Updates IpField to be searchable when only `doc_values` are enabled ([#11508](https://github.com/opensearch-project/OpenSearch/pull/11508))
+- Workaround for https://bugs.openjdk.org/browse/JDK-8323659 regression, introduced in JDK-21.0.2 ([#11968](https://github.com/opensearch-project/OpenSearch/pull/11968))
 
 ### Deprecated
 
@@ -237,6 +241,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix issue when calling Delete PIT endpoint and no PITs exist ([#11711](https://github.com/opensearch-project/OpenSearch/pull/11711))
 - Fix tracing context propagation for local transport instrumentation ([#11490](https://github.com/opensearch-project/OpenSearch/pull/11490))
 - Fix parsing of single line comments in `lang-painless` ([#11815](https://github.com/opensearch-project/OpenSearch/issues/11815))
+- Fix memory leak issue in ReorganizingLongHash ([#11953](https://github.com/opensearch-project/OpenSearch/issues/11953))
 
 ### Security
 

build.gradle

@@ -54,7 +54,7 @@ plugins {
   id 'lifecycle-base'
   id 'opensearch.docker-support'
   id 'opensearch.global-build-info'
-  id "com.diffplug.spotless" version "6.23.2" apply false
+  id "com.diffplug.spotless" version "6.24.0" apply false
   id "org.gradle.test-retry" version "1.5.4" apply false
   id "test-report-aggregation"
   id 'jacoco-report-aggregation'

buildSrc/build.gradle

@@ -118,7 +118,7 @@ dependencies {
   api 'com.avast.gradle:gradle-docker-compose-plugin:0.17.6'
   api "org.yaml:snakeyaml:${props.getProperty('snakeyaml')}"
   api 'org.apache.maven:maven-model:3.9.6'
-  api 'com.networknt:json-schema-validator:1.1.0'
+  api 'com.networknt:json-schema-validator:1.2.0'
   api 'org.jruby.jcodings:jcodings:1.0.58'
   api 'org.jruby.joni:joni:2.2.1'
   api "com.fasterxml.jackson.core:jackson-databind:${props.getProperty('jackson_databind')}"

modules/ingest-common/src/test/java/org/opensearch/ingest/common/CopyProcessorTests.java

@@ -14,6 +14,9 @@
 import org.opensearch.ingest.TestTemplateService;
 import org.opensearch.test.OpenSearchTestCase;
 
+import java.util.List;
+import java.util.Map;
+
 import static org.hamcrest.Matchers.equalTo;
 
 public class CopyProcessorTests extends OpenSearchTestCase {
@@ -26,8 +29,7 @@ public void testCopyExistingField() throws Exception {
         processor.execute(ingestDocument);
         assertThat(ingestDocument.hasField(targetFieldName), equalTo(true));
         Object sourceValue = ingestDocument.getFieldValue(sourceFieldName, Object.class);
-        assertThat(ingestDocument.getFieldValue(targetFieldName, Object.class), equalTo(sourceValue));
-        assertThat(ingestDocument.getFieldValue(sourceFieldName, Object.class), equalTo(sourceValue));
+        assertDeepCopiedObjectEquals(ingestDocument.getFieldValue(targetFieldName, Object.class), sourceValue);
 
         Processor processorWithEmptyTarget = createCopyProcessor(sourceFieldName, "", false, false, false);
         assertThrows(
@@ -75,7 +77,7 @@ public void testCopyWithRemoveSource() throws Exception {
         Processor processor = createCopyProcessor(sourceFieldName, targetFieldName, false, true, false);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.hasField(targetFieldName), equalTo(true));
-        assertThat(ingestDocument.getFieldValue(targetFieldName, Object.class), equalTo(sourceValue));
+        assertDeepCopiedObjectEquals(ingestDocument.getFieldValue(targetFieldName, Object.class), sourceValue);
         assertThat(ingestDocument.hasField(sourceFieldName), equalTo(false));
     }
 
@@ -97,12 +99,30 @@ public void testCopyToExistingField() throws Exception {
         Processor processorWithTargetNullValue = createCopyProcessor(sourceFieldName, targetFieldWithNullValue, false, false, false);
         processorWithTargetNullValue.execute(ingestDocument);
         assertThat(ingestDocument.hasField(targetFieldWithNullValue), equalTo(true));
-        assertThat(ingestDocument.getFieldValue(targetFieldWithNullValue, Object.class), equalTo(sourceValue));
+        assertDeepCopiedObjectEquals(ingestDocument.getFieldValue(targetFieldWithNullValue, Object.class), sourceValue);
 
         Processor processorWithOverrideTargetIsTrue = createCopyProcessor(sourceFieldName, targetFieldName, false, false, true);
         processorWithOverrideTargetIsTrue.execute(ingestDocument);
         assertThat(ingestDocument.hasField(targetFieldName), equalTo(true));
-        assertThat(ingestDocument.getFieldValue(targetFieldName, Object.class), equalTo(sourceValue));
+        assertDeepCopiedObjectEquals(ingestDocument.getFieldValue(targetFieldName, Object.class), sourceValue);
+    }
+
+    @SuppressWarnings("unchecked")
+    private static void assertDeepCopiedObjectEquals(Object expected, Object actual) {
+        if (expected instanceof Map) {
+            Map<String, Object> expectedMap = (Map<String, Object>) expected;
+            Map<String, Object> actualMap = (Map<String, Object>) actual;
+            assertEquals(expectedMap.size(), actualMap.size());
+            for (Map.Entry<String, Object> expectedEntry : expectedMap.entrySet()) {
+                assertDeepCopiedObjectEquals(expectedEntry.getValue(), actualMap.get(expectedEntry.getKey()));
+            }
+        } else if (expected instanceof List) {
+            assertArrayEquals(((List<?>) expected).toArray(), ((List<?>) actual).toArray());
+        } else if (expected instanceof byte[]) {
+            assertArrayEquals((byte[]) expected, (byte[]) actual);
+        } else {
+            assertEquals(expected, actual);
+        }
     }
 
     private static Processor createCopyProcessor(

plugins/discovery-gce/build.gradle

@@ -24,7 +24,7 @@ versions << [
 dependencies {
   api "com.google.apis:google-api-services-compute:v1-rev160-${versions.google}"
   api "com.google.api-client:google-api-client:${versions.google}"
-  api "com.google.oauth-client:google-oauth-client:1.34.1"
+  api "com.google.oauth-client:google-oauth-client:1.35.0"
   api "com.google.http-client:google-http-client:${versions.google}"
   api "com.google.http-client:google-http-client-jackson2:${versions.google}"
   api 'com.google.code.findbugs:jsr305:3.0.2'

plugins/discovery-gce/licenses/google-oauth-client-1.35.0.jar.sha1

@@ -0,0 +1 @@
+2f52003156e40ba8be5f349a2716a77428896e69

plugins/repository-gcs/build.gradle

@@ -66,7 +66,7 @@ dependencies {
   api "com.google.auth:google-auth-library-credentials:${versions.google_auth}"
   api "com.google.auth:google-auth-library-oauth2-http:${versions.google_auth}"
 
-  api 'com.google.cloud:google-cloud-core:2.5.10'
+  api 'com.google.cloud:google-cloud-core:2.30.0'
   api 'com.google.cloud:google-cloud-core-http:2.23.0'
   api 'com.google.cloud:google-cloud-storage:1.113.1'
 

plugins/repository-gcs/licenses/google-cloud-core-2.30.0.jar.sha1

@@ -0,0 +1 @@
+b48ea27cbdccd5f225d8a35ea28e2cd01c25918b

server/src/main/java/org/opensearch/common/util/ReorganizingLongHash.java

@@ -118,10 +118,17 @@ public ReorganizingLongHash(final long initialCapacity, final float loadFactor,
         mask = capacity - 1;
         grow = (long) (capacity * loadFactor);
         size = 0;
-
-        table = bigArrays.newLongArray(capacity, false);
-        table.fill(0, capacity, -1);  // -1 represents an empty slot
-        keys = bigArrays.newLongArray(initialCapacity, false);
+        try {
+            table = bigArrays.newLongArray(capacity, false);
+            table.fill(0, capacity, -1);  // -1 represents an empty slot
+            keys = bigArrays.newLongArray(initialCapacity, false);
+        } finally {
+            if (table == null || keys == null) {
+                // it's important to close the arrays initialized above to prevent memory leak
+                // refer: https://github.com/opensearch-project/OpenSearch/issues/10154
+                Releasables.closeWhileHandlingException(table, keys);
+            }
+        }
     }
 
     /**

server/src/main/java/org/opensearch/common/util/concurrent/OpenSearchExecutors.java

@@ -447,6 +447,30 @@ public boolean offer(E e) {
             }
         }
 
+        /**
+         * Workaround for https://bugs.openjdk.org/browse/JDK-8323659 regression, introduced in JDK-21.0.2.
+         */
+        @Override
+        public void put(E e) {
+            super.offer(e);
+        }
+
+        /**
+         * Workaround for https://bugs.openjdk.org/browse/JDK-8323659 regression, introduced in JDK-21.0.2.
+         */
+        @Override
+        public boolean offer(E e, long timeout, TimeUnit unit) {
+            return super.offer(e);
+        }
+
+        /**
+         * Workaround for https://bugs.openjdk.org/browse/JDK-8323659 regression, introduced in JDK-21.0.2.
+         */
+        @Override
+        public boolean add(E e) {
+            return super.offer(e);
+        }
+
     }
 
     /**

server/src/main/java/org/opensearch/rest/action/admin/cluster/RestRemoteClusterInfoAction.java

@@ -53,7 +53,7 @@ public final class RestRemoteClusterInfoAction extends BaseRestHandler {
 
     @Override
     public List<Route> routes() {
-        return singletonList(new Route(GET, "_remote/info"));
+        return singletonList(new Route(GET, "/_remote/info"));
     }
 
     @Override

server/src/main/java/org/opensearch/rest/action/admin/indices/RestPutMappingAction.java

@@ -66,10 +66,10 @@ public class RestPutMappingAction extends BaseRestHandler {
     public List<Route> routes() {
         return unmodifiableList(
             asList(
-                new Route(POST, "/{index}/_mapping/"),
-                new Route(PUT, "/{index}/_mapping/"),
-                new Route(POST, "/{index}/_mappings/"),
-                new Route(PUT, "/{index}/_mappings/")
+                new Route(POST, "/{index}/_mapping"),
+                new Route(PUT, "/{index}/_mapping"),
+                new Route(POST, "/{index}/_mappings"),
+                new Route(PUT, "/{index}/_mappings")
             )
         );
     }