HotToWarmTieringService changes and changes in shard balancer to tier shards

.ci/bwcVersions

@@ -36,3 +36,4 @@ BWC_VERSION:
   - "2.15.0"
   - "2.15.1"
   - "2.16.0"
+  - "2.17.0"

.github/CODEOWNERS

@@ -13,15 +13,25 @@
 # Default ownership for all repo files
 * @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
 
+/modules/lang-painless/ @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
+/modules/parent-join/ @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
 /modules/transport-netty4/ @peternied
 
 /plugins/identity-shiro/ @peternied
 
+/server/src/internalClusterTest/java/org/opensearch/index/ @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
+/server/src/internalClusterTest/java/org/opensearch/search/ @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
+
 /server/src/main/java/org/opensearch/extensions/ @peternied
 /server/src/main/java/org/opensearch/identity/ @peternied
-/server/src/main/java/org/opensearch/threadpool/ @peternied
+/server/src/main/java/org/opensearch/index/ @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
+/server/src/main/java/org/opensearch/search/ @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
+/server/src/main/java/org/opensearch/threadpool/ @jed326 @peternied
 /server/src/main/java/org/opensearch/transport/ @peternied
 
-/.github/ @peternied
+/server/src/test/java/org/opensearch/index/ @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
+/server/src/test/java/org/opensearch/search/ @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah
+
+/.github/ @jed326 @peternied
 
 /MAINTAINERS.md @anasalkouz @andrross @ashking94 @Bukhtawar @CEHENKLE @dblock @dbwiddis @gaobinlong @gbbafna @jed326 @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @peternied @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @VachaShah

.github/benchmark-configs.json

@@ -14,7 +14,8 @@
     "cluster_configuration": {
       "size": "Single-Node",
       "data_instance_config": "4vCPU, 32G Mem, 16G Heap"
-    }
+    },
+    "baseline_cluster_config": "x64-r5.xlarge-single-node-1-shard-0-replica-baseline"
   },
   "id_2": {
     "description": "Indexing only configuration for HTTP_LOGS workload",
@@ -30,7 +31,8 @@
     "cluster_configuration": {
       "size": "Single-Node",
       "data_instance_config": "4vCPU, 32G Mem, 16G Heap"
-    }
+    },
+    "baseline_cluster_config": "x64-r5.xlarge-single-node-1-shard-0-replica-baseline"
   },
   "id_3": {
     "description": "Search only test-procedure for NYC_TAXIS, uses snapshot to restore the data for OS-3.0.0",
@@ -46,7 +48,8 @@
     "cluster_configuration": {
       "size": "Single-Node",
       "data_instance_config": "4vCPU, 32G Mem, 16G Heap"
-    }
+    },
+    "baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
   },
   "id_4": {
     "description": "Search only test-procedure for HTTP_LOGS, uses snapshot to restore the data for OS-3.0.0",
@@ -62,10 +65,11 @@
     "cluster_configuration": {
       "size": "Single-Node",
       "data_instance_config": "4vCPU, 32G Mem, 16G Heap"
-    }
+    },
+    "baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
   },
   "id_5": {
-    "description": "Search only test-procedure for HTTP_LOGS, uses snapshot to restore the data for OS-3.0.0",
+    "description": "Search only test-procedure for big5, uses snapshot to restore the data for OS-3.0.0",
     "supported_major_versions": ["3"],
     "cluster-benchmark-configs": {
       "SINGLE_NODE_CLUSTER": "true",
@@ -78,7 +82,8 @@
     "cluster_configuration": {
       "size": "Single-Node",
       "data_instance_config": "4vCPU, 32G Mem, 16G Heap"
-    }
+    },
+    "baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
   },
   "id_6": {
     "description": "Search only test-procedure for NYC_TAXIS, uses snapshot to restore the data for OS-2.x",
@@ -94,7 +99,8 @@
     "cluster_configuration": {
       "size": "Single-Node",
       "data_instance_config": "4vCPU, 32G Mem, 16G Heap"
-    }
+    },
+    "baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
   },
   "id_7": {
     "description": "Search only test-procedure for HTTP_LOGS, uses snapshot to restore the data for OS-2.x",
@@ -110,10 +116,11 @@
     "cluster_configuration": {
       "size": "Single-Node",
       "data_instance_config": "4vCPU, 32G Mem, 16G Heap"
-    }
+    },
+    "baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
   },
   "id_8": {
-    "description": "Search only test-procedure for HTTP_LOGS, uses snapshot to restore the data for OS-2.x",
+    "description": "Search only test-procedure for big5, uses snapshot to restore the data for OS-2.x",
     "supported_major_versions": ["2"],
     "cluster-benchmark-configs": {
       "SINGLE_NODE_CLUSTER": "true",
@@ -126,7 +133,8 @@
     "cluster_configuration": {
       "size": "Single-Node",
       "data_instance_config": "4vCPU, 32G Mem, 16G Heap"
-    }
+    },
+    "baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
   },
   "id_9": {
     "description": "Indexing and search configuration for pmc workload",
@@ -141,7 +149,8 @@
     "cluster_configuration": {
       "size": "Single-Node",
       "data_instance_config": "4vCPU, 32G Mem, 16G Heap"
-    }
+    },
+    "baseline_cluster_config": "x64-r5.xlarge-single-node-1-shard-0-replica-baseline"
   },
   "id_10": {
     "description": "Indexing only configuration for stack-overflow workload",
@@ -156,6 +165,7 @@
     "cluster_configuration": {
       "size": "Single-Node",
       "data_instance_config": "4vCPU, 32G Mem, 16G Heap"
-    }
+    },
+    "baseline_cluster_config": "x64-r5.xlarge-single-node-1-shard-0-replica-baseline"
   }
 }

.github/workflows/add-performance-comment.yml

@@ -6,14 +6,17 @@ on:
 
 jobs:
   add-comment:
-    if: github.event.label.name == 'Performance'
+    if: |
+      github.event.label.name == 'Performance' ||
+      github.event.label.name == 'Search:Performance' ||
+      github.event.label.name == 'Indexing:Performance'
     runs-on: ubuntu-latest
     permissions:
       pull-requests: write
 
     steps:
       - name: Add comment to PR
-        uses: actions/github-script@v6
+        uses: actions/github-script@v7
         with:
           github-token: ${{secrets.GITHUB_TOKEN}}
           script: |

.github/workflows/benchmark-pull-request.yml

@@ -25,7 +25,7 @@ jobs:
           echo "USER_TAGS=pull_request_number:${{ github.event.issue.number }},repository:OpenSearch" >> $GITHUB_ENV
       - name: Check comment format
         id: check_comment
-        uses: actions/github-script@v6
+        uses: actions/github-script@v7
         with:
           script: |
             const fs = require('fs');
@@ -60,9 +60,11 @@ jobs:
             for (const [key, value] of Object.entries(clusterBenchmarkConfigs)) {
               core.exportVariable(key, value);
             }
+            if (benchmarkConfigs[configId].hasOwnProperty('baseline_cluster_config')) {
+              core.exportVariable('BASELINE_CLUSTER_CONFIG', benchmarkConfigs[configId]['baseline_cluster_config']);
       - name: Post invalid format comment
         if: steps.check_comment.outputs.invalid == 'true'
-        uses: actions/github-script@v6
+        uses: actions/github-script@v7
         with:
           github-token: ${{secrets.GITHUB_TOKEN}}
           script: |
@@ -77,18 +79,6 @@ jobs:
         run: |
           echo "Invalid comment format detected. Failing the workflow."
           exit 1
-      - id: get_approvers
-        run: |
-          echo "approvers=$(cat .github/CODEOWNERS | grep '^\*'  | tr -d '* ' | sed 's/@/,/g' | sed 's/,//1')" >> $GITHUB_OUTPUT
-      - uses: trstringer/manual-approval@v1
-        if: (!contains(steps.get_approvers.outputs.approvers, github.event.comment.user.login))
-        with:
-          secret: ${{ github.TOKEN }}
-          approvers: ${{ steps.get_approvers.outputs.approvers }}
-          minimum-approvals: 1
-          issue-title: 'Request to approve/deny benchmark run for PR #${{ env.PR_NUMBER }}'
-          issue-body: "Please approve or deny the benchmark run for PR #${{ env.PR_NUMBER }}"
-          exclude-workflow-initiator-as-approver: false
       - name: Get PR Details
         id: get_pr
         uses: actions/github-script@v7
@@ -106,24 +96,36 @@ jobs:
 
             return {
               "headRepoFullName": pull_request.head.repo.full_name,
-              "headRef": pull_request.head.ref
+              "headRefSha": pull_request.head.sha
             };
       - name: Set pr details env vars
         run: |
           echo '${{ steps.get_pr.outputs.result }}' | jq -r '.headRepoFullName'
-          echo '${{ steps.get_pr.outputs.result }}' | jq -r '.headRef'
+          echo '${{ steps.get_pr.outputs.result }}' | jq -r '.headRefSha'
           headRepo=$(echo '${{ steps.get_pr.outputs.result }}' | jq -r '.headRepoFullName')
-          headRef=$(echo '${{ steps.get_pr.outputs.result }}' | jq -r '.headRef')
+          headRefSha=$(echo '${{ steps.get_pr.outputs.result }}' | jq -r '.headRefSha')
           echo "prHeadRepo=$headRepo" >> $GITHUB_ENV
-          echo "prHeadRef=$headRef" >> $GITHUB_ENV
+          echo "prHeadRefSha=$headRefSha" >> $GITHUB_ENV
+      - id: get_approvers
+        run: |
+          echo "approvers=$(cat .github/CODEOWNERS | grep '^\*'  | tr -d '* ' | sed 's/@/,/g' | sed 's/,//1')" >> $GITHUB_OUTPUT
+      - uses: trstringer/manual-approval@v1
+        if: (!contains(steps.get_approvers.outputs.approvers, github.event.comment.user.login))
+        with:
+          secret: ${{ github.TOKEN }}
+          approvers: ${{ steps.get_approvers.outputs.approvers }}
+          minimum-approvals: 1
+          issue-title: 'Request to approve/deny benchmark run for PR #${{ env.PR_NUMBER }}'
+          issue-body: "Please approve or deny the benchmark run for PR #${{ env.PR_NUMBER }}"
+          exclude-workflow-initiator-as-approver: false
       - name: Checkout PR Repo
         uses: actions/checkout@v4
         with:
           repository: ${{ env.prHeadRepo }}
-          ref: ${{ env.prHeadRef }}
+          ref: ${{ env.prHeadRefSha }}
           token: ${{ secrets.GITHUB_TOKEN }}
       - name: Setup Java
-        uses: actions/setup-java@v1
+        uses: actions/setup-java@v4
         with:
           java-version: 21
       - name: Build and Assemble OpenSearch from PR
@@ -150,7 +152,7 @@ jobs:
           cat $GITHUB_ENV
           bash opensearch-build/scripts/benchmark/benchmark-pull-request.sh ${{ secrets.JENKINS_PR_BENCHMARK_GENERIC_WEBHOOK_TOKEN }}
       - name: Update PR with Job Url
-        uses: actions/github-script@v6
+        uses: actions/github-script@v7
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           script: |

.github/workflows/maintainer-approval.yml

@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - id: find-maintainers
-        uses: actions/github-script@v7.0.1
+        uses: actions/github-script@v7
         with:
           github-token: ${{ secrets.GITHUB_TOKEN }}
           result-encoding: string

.github/workflows/precommit.yml

@@ -8,7 +8,7 @@ jobs:
     strategy:
       matrix:
         java: [ 11, 17, 21 ]
-        os: [ubuntu-latest, windows-latest, macos-13]
+        os: [ubuntu-latest, windows-latest, macos-latest, macos-13]
     steps:
       - uses: actions/checkout@v4
       - name: Set up JDK ${{ matrix.java }}

.github/workflows/triage.yml

@@ -9,7 +9,7 @@ jobs:
     if: github.repository == 'opensearch-project/OpenSearch'
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/github-script@v7.0.1
+      - uses: actions/github-script@v7
         with:
           script: |
             const { issue, repository } = context.payload;

.github/workflows/version.yml

@@ -129,7 +129,7 @@ jobs:
 
       - name: Create tracking issue
         id: create-issue
-        uses: actions/github-script@v7.0.1
+        uses: actions/github-script@v7
         with:
           script: |
             const body = `

CHANGELOG-3.0.md

@@ -43,6 +43,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Remove LegacyESVersion.V_7_10_ Constants ([#5018](https://github.com/opensearch-project/OpenSearch/pull/5018))
 - Remove Version.V_1_ Constants ([#5021](https://github.com/opensearch-project/OpenSearch/pull/5021))
 - Remove custom Map, List and Set collection classes ([#6871](https://github.com/opensearch-project/OpenSearch/pull/6871))
+- Remove `index.store.hybrid.mmap.extensions` setting in favor of `index.store.hybrid.nio.extensions` setting ([#9392](https://github.com/opensearch-project/OpenSearch/pull/9392))
 
 ### Fixed
 - Fix 'org.apache.hc.core5.http.ParseException: Invalid protocol version' under JDK 16+ ([#4827](https://github.com/opensearch-project/OpenSearch/pull/4827))