diff --git a/.ci/bwcVersions b/.ci/bwcVersions
index 78d8796c624d7..e490c6ae6b4ea 100644
--- a/.ci/bwcVersions
+++ b/.ci/bwcVersions
@@ -30,4 +30,6 @@ BWC_VERSION:
   - "2.12.0"
   - "2.12.1"
   - "2.13.0"
+  - "2.13.1"
   - "2.14.0"
+  - "2.15.0"
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
index bb12121cd3d8f..40c8fb2e01dd7 100644
--- a/.github/CODEOWNERS
+++ b/.github/CODEOWNERS
@@ -11,7 +11,7 @@
 #   3. Use the command palette to run the CODEOWNERS: Show owners of current file command, which will display all code owners for the current file.
 
 # Default ownership for all repo files
-* @abbashus @adnapibar @anasalkouz @andrross @Bukhtawar @CEHENKLE @dblock @dbwiddis @dreamer-89 @gbbafna @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @ryanbogan @sachinpkale @saratvemulapalli @setiah @shwetathareja @sohami @tlfeng @VachaShah
+* @anasalkouz @andrross @Bukhtawar @CEHENKLE @dblock @dbwiddis @dreamer-89 @gbbafna @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @tlfeng @VachaShah
 
 /modules/transport-netty4/ @peternied
 
@@ -24,4 +24,4 @@
 
 /.github/ @peternied
 
-/MAINTAINERS.md @abbashus @adnapibar @anasalkouz @andrross @Bukhtawar @CEHENKLE @dblock @dbwiddis @dreamer-89 @gbbafna @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @peternied @reta @Rishikesh1159 @ryanbogan @sachinpkale @saratvemulapalli @setiah @shwetathareja @sohami @tlfeng @VachaShah
+/MAINTAINERS.md @anasalkouz @andrross @Bukhtawar @CEHENKLE @dblock @dbwiddis @dreamer-89 @gbbafna @kotwanikunal @mch2 @msfroh @nknize @owaiskazi19 @peternied @reta @Rishikesh1159 @sachinpkale @saratvemulapalli @shwetathareja @sohami @tlfeng @VachaShah
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
index 908a032bf833e..6281fa0af3e36 100644
--- a/.github/pull_request_template.md
+++ b/.github/pull_request_template.md
@@ -17,6 +17,7 @@ Resolves #[Issue number to be closed when this PR is merged]
   - [ ] All tests pass
 - [ ] New functionality has been documented.
   - [ ] New functionality has javadoc added
+- [ ] API changes companion pull request [created](https://github.com/opensearch-project/opensearch-api-specification/blob/main/DEVELOPER_GUIDE.md).
 - [ ] Failing checks are inspected and point to the corresponding known issue(s) (See: [Troubleshooting Failing Builds](../blob/main/CONTRIBUTING.md#troubleshooting-failing-builds))
 - [ ] Commits are signed per the DCO using --signoff
 - [ ] Commit changes are listed out in CHANGELOG.md file (See: [Changelog](../blob/main/CONTRIBUTING.md#changelog))
diff --git a/.github/workflows/assemble.yml b/.github/workflows/assemble.yml
index 382105364c048..d18170e9ea6b7 100644
--- a/.github/workflows/assemble.yml
+++ b/.github/workflows/assemble.yml
@@ -8,7 +8,7 @@ jobs:
     strategy:
       matrix:
         java: [ 11, 17, 21 ]
-        os: [ubuntu-latest, windows-latest, macos-latest]
+        os: [ubuntu-latest, windows-latest, macos-13]
     steps:
       - uses: actions/checkout@v4
       - name: Set up JDK ${{ matrix.java }}
@@ -18,10 +18,7 @@ jobs:
           distribution: temurin
       - name: Setup docker (missing on MacOS)
         if: runner.os == 'macos'
-        run: |
-          brew install docker
-          colima start
-          sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock
+        uses: douglascamata/setup-docker-macos-action@main
       - name: Run Gradle (assemble)
         run: |
           ./gradlew assemble --parallel --no-build-cache -PDISABLE_BUILD_CACHE
diff --git a/.github/workflows/changelog_verifier.yml b/.github/workflows/changelog_verifier.yml
index 9456fbf8b4ca0..cf9343c2c3aac 100644
--- a/.github/workflows/changelog_verifier.yml
+++ b/.github/workflows/changelog_verifier.yml
@@ -1,7 +1,7 @@
 name: "Changelog Verifier"
 on:
   pull_request:
-    types: [opened, edited, review_requested, synchronize, reopened, ready_for_review, labeled, unlabeled]
+    types: [opened, synchronize, reopened, ready_for_review, labeled, unlabeled]
 
 jobs:
   # Enforces the update of a changelog file on every pull request
@@ -13,7 +13,32 @@ jobs:
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           ref: ${{ github.event.pull_request.head.sha }}
-
       - uses: dangoslen/changelog-enforcer@v3
+        id: verify-changelog-3x
+        with:
+          skipLabels: "autocut, skip-changelog"
+          changeLogPath: 'CHANGELOG-3.0.md'
+        continue-on-error: true
+      - uses: dangoslen/changelog-enforcer@v3
+        id: verify-changelog
         with:
           skipLabels: "autocut, skip-changelog"
+          changeLogPath: 'CHANGELOG.md'
+        continue-on-error: true
+      - run: |
+          # The check was possibly skipped leading to success for both the jobs
+          if [[ ${{ steps.verify-changelog-3x.outcome }} == 'success' && ${{ steps.verify-changelog.outcome }} == 'success' ]]; then
+            exit 0
+          fi
+          
+          if [[ ${{ steps.verify-changelog-3x.outcome }} == 'failure' && ${{ steps.verify-changelog.outcome }} == 'failure' ]]; then
+            echo "error: Please ensure a changelog entry exists in CHANGELOG.md or CHANGELOG-3.0.md"
+            exit 1
+          fi
+          
+          # Concatenates the labels and checks if the string contains "backport"
+          has_backport_label=${{ contains(join(github.event.pull_request.labels.*.name, ', '), 'backport')}}
+          if [[ ${{ steps.verify-changelog.outcome }} == 'success' && $has_backport_label == false ]]; then
+            echo "error: Please make sure that the PR has a backport label associated with it when making an entry to the CHANGELOG.md file"
+            exit 1
+          fi
diff --git a/.github/workflows/check-compatibility.yml b/.github/workflows/check-compatibility.yml
deleted file mode 100644
index b2f22a90938cc..0000000000000
--- a/.github/workflows/check-compatibility.yml
+++ /dev/null
@@ -1,69 +0,0 @@
----
-name: Check Compatibility
-
-on:
-  pull_request_target
-
-jobs:
-  check-compatibility:
-    if: github.repository == 'opensearch-project/OpenSearch'
-    permissions:
-      contents: read
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{ github.event.pull_request.head.sha }}
-
-      - name: Increase swapfile
-        run: |
-          sudo swapoff -a
-          sudo fallocate -l 10G /swapfile
-          sudo chmod 600 /swapfile
-          sudo mkswap /swapfile
-          sudo swapon /swapfile
-          sudo swapon --show
-
-      - name: Run compatibility task
-        run: ./gradlew checkCompatibility -i | tee $HOME/gradlew-check.out
-
-      - name: Get results
-        run: |
-          echo '## Compatibility status:' > "${{ github.workspace }}/results.txt"
-          echo "Checks if related components are compatible with change $(git rev-parse --short HEAD)" >> "${{ github.workspace }}/results.txt"
-          echo "### Incompatible components" >> "${{ github.workspace }}/results.txt" && grep -e 'Incompatible component' $HOME/gradlew-check.out | sed -e 's/Incompatible component: \[\(.*\)\]/- \1/' >> "${{ github.workspace }}/results.txt"
-          echo "### Skipped components"      >> "${{ github.workspace }}/results.txt" && grep -e 'Skipped component'      $HOME/gradlew-check.out | sed -e 's/Skipped component: \[\(.*\)\]/- \1/'      >> "${{ github.workspace }}/results.txt"
-          echo "### Compatible components"   >> "${{ github.workspace }}/results.txt" && grep -e 'Compatible component'   $HOME/gradlew-check.out | sed -e 's/Compatible component: \[\(.*\)\]/- \1/'   >> "${{ github.workspace }}/results.txt"
-
-      - name: Upload results
-        uses: actions/upload-artifact@v4
-        with:
-          name: results.txt
-          path: ${{ github.workspace }}/results.txt
-
-  add-comment:
-    needs: [check-compatibility]
-    permissions:
-      pull-requests: write
-    runs-on: ubuntu-latest
-    steps:
-      - name: Download results
-        uses: actions/download-artifact@v4
-        with:
-          name: results.txt
-
-      - name: Find Comment
-        uses: peter-evans/find-comment@v3
-        id: fc
-        with:
-          issue-number: ${{ github.event.number }}
-          comment-author: 'github-actions[bot]'
-          body-includes: 'Compatibility status:'
-
-      - name: Add comment on the PR
-        uses: peter-evans/create-or-update-comment@v4
-        with:
-          comment-id: ${{ steps.fc.outputs.comment-id }}
-          issue-number: ${{ github.event.number }}
-          body-path: results.txt
-          edit-mode: replace
diff --git a/.github/workflows/detect-breaking-change.yml b/.github/workflows/detect-breaking-change.yml
new file mode 100644
index 0000000000000..e5d3fddbd36f5
--- /dev/null
+++ b/.github/workflows/detect-breaking-change.yml
@@ -0,0 +1,29 @@
+name: "Detect Breaking Changes"
+on:
+  pull_request:
+    branches-ignore:
+      - main # This branch represents a to-be-released version of OpenSearch where breaking changes are allowed
+
+jobs:
+  detect-breaking-change:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-java@v4
+      with:
+        distribution: temurin # Temurin is a distribution of adoptium
+        java-version: 21
+    - uses: gradle/gradle-build-action@v3
+      with:
+        cache-disabled: true
+        arguments: japicmp
+        gradle-version: 8.7
+        build-root-directory: server
+    - if: failure()
+      run: cat server/build/reports/java-compatibility/report.txt
+    - if: failure()
+      uses: actions/upload-artifact@v4
+      with:
+        name: java-compatibility-report.html
+        path: ${{ github.workspace }}/server/build/reports/java-compatibility/report.html
+  
\ No newline at end of file
diff --git a/.github/workflows/links.yml b/.github/workflows/links.yml
index 1c83821e22804..8f628fcd78148 100644
--- a/.github/workflows/links.yml
+++ b/.github/workflows/links.yml
@@ -13,7 +13,7 @@ jobs:
       - uses: actions/checkout@v4
       - name: lychee Link Checker
         id: lychee
-        uses: lycheeverse/lychee-action@v1.9.3
+        uses: lycheeverse/lychee-action@v1.10.0
         with:
           args: --accept=200,403,429 --exclude-mail **/*.html **/*.md **/*.txt **/*.json --exclude-file .lychee.excludes
           fail: true
diff --git a/.github/workflows/precommit.yml b/.github/workflows/precommit.yml
index 800aacec98516..95ca49ac9cb43 100644
--- a/.github/workflows/precommit.yml
+++ b/.github/workflows/precommit.yml
@@ -8,7 +8,7 @@ jobs:
     strategy:
       matrix:
         java: [ 11, 17, 21 ]
-        os: [ubuntu-latest, windows-latest, macos-latest]
+        os: [ubuntu-latest, windows-latest, macos-13]
     steps:
       - uses: actions/checkout@v4
       - name: Set up JDK ${{ matrix.java }}
diff --git a/.github/workflows/pull-request-checks.yml b/.github/workflows/pull-request-checks.yml
index 7efcf529588ed..a62ea9cfa179b 100644
--- a/.github/workflows/pull-request-checks.yml
+++ b/.github/workflows/pull-request-checks.yml
@@ -18,6 +18,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: peternied/check-pull-request-description-checklist@v1.1
+        if: github.actor != 'dependabot[bot]'
         with:
           checklist-items: |
             New functionality includes testing.
diff --git a/.github/workflows/wrapper.yml b/.github/workflows/wrapper.yml
index dcf2a09717d28..cc9fb2d78f919 100644
--- a/.github/workflows/wrapper.yml
+++ b/.github/workflows/wrapper.yml
@@ -8,4 +8,4 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: gradle/wrapper-validation-action@v2
+      - uses: gradle/wrapper-validation-action@v3
diff --git a/CHANGELOG-3.0.md b/CHANGELOG-3.0.md
new file mode 100644
index 0000000000000..964383078c38d
--- /dev/null
+++ b/CHANGELOG-3.0.md
@@ -0,0 +1,53 @@
+# CHANGELOG
+All notable changes to this project are documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). See the [CONTRIBUTING guide](./CONTRIBUTING.md#Changelog) for instructions on how to add changelog entries.
+
+## [Unreleased 3.0]
+### Added
+- Support for HTTP/2 (server-side) ([#3847](https://github.com/opensearch-project/OpenSearch/pull/3847))
+- Allow mmap to use new JDK-19 preview APIs in Apache Lucene 9.4+ ([#5151](https://github.com/opensearch-project/OpenSearch/pull/5151))
+- Add events correlation engine plugin ([#6854](https://github.com/opensearch-project/OpenSearch/issues/6854))
+- Implement on behalf of token passing for extensions ([#8679](https://github.com/opensearch-project/OpenSearch/pull/8679), [#10664](https://github.com/opensearch-project/OpenSearch/pull/10664))
+- Provide service accounts tokens to extensions ([#9618](https://github.com/opensearch-project/OpenSearch/pull/9618))
+- GHA to verify checklist items completion in PR descriptions ([#10800](https://github.com/opensearch-project/OpenSearch/pull/10800))
+- Allow to pass the list settings through environment variables (like [], ["a", "b", "c"], ...) ([#10625](https://github.com/opensearch-project/OpenSearch/pull/10625))
+- Views, simplify data access and manipulation by providing a virtual layer over one or more indices ([#11957](https://github.com/opensearch-project/OpenSearch/pull/11957))
+
+### Dependencies
+
+### Changed
+- Migrate client transports to Apache HttpClient / Core 5.x ([#4459](https://github.com/opensearch-project/OpenSearch/pull/4459))
+- Change http code on create index API with bad input raising NotXContentException from 500 to 400 ([#4773](https://github.com/opensearch-project/OpenSearch/pull/4773))
+- Improve summary error message for invalid setting updates ([#4792](https://github.com/opensearch-project/OpenSearch/pull/4792))
+- Return 409 Conflict HTTP status instead of 503 on failure to concurrently execute snapshots ([#8986](https://github.com/opensearch-project/OpenSearch/pull/5855))
+- Add task completion count in search backpressure stats API ([#10028](https://github.com/opensearch-project/OpenSearch/pull/10028/))
+- Deprecate CamelCase `PathHierarchy` tokenizer name in favor to lowercase `path_hierarchy` ([#10894](https://github.com/opensearch-project/OpenSearch/pull/10894))
+- Breaking change: Do not request "search_pipelines" metrics by default in NodesInfoRequest ([#12497](https://github.com/opensearch-project/OpenSearch/pull/12497))
+
+### Deprecated
+
+### Removed
+- Remove deprecated code to add node name into log pattern of log4j property file ([#4568](https://github.com/opensearch-project/OpenSearch/pull/4568))
+- Unused object and import within TransportClusterAllocationExplainAction ([#4639](https://github.com/opensearch-project/OpenSearch/pull/4639))
+- Remove LegacyESVersion.V_7_0_* and V_7_1_* Constants ([#2768](https://https://github.com/opensearch-project/OpenSearch/pull/2768))
+- Remove LegacyESVersion.V_7_2_ and V_7_3_ Constants ([#4702](https://github.com/opensearch-project/OpenSearch/pull/4702))
+- Always auto release the flood stage block ([#4703](https://github.com/opensearch-project/OpenSearch/pull/4703))
+- Remove LegacyESVersion.V_7_4_ and V_7_5_ Constants ([#4704](https://github.com/opensearch-project/OpenSearch/pull/4704))
+- Remove Legacy Version support from Snapshot/Restore Service ([#4728](https://github.com/opensearch-project/OpenSearch/pull/4728))
+- Remove deprecated serialization logic from pipeline aggs ([#4847](https://github.com/opensearch-project/OpenSearch/pull/4847))
+- Remove unused private methods ([#4926](https://github.com/opensearch-project/OpenSearch/pull/4926))
+- Remove LegacyESVersion.V_7_8_ and V_7_9_ Constants ([#4855](https://github.com/opensearch-project/OpenSearch/pull/4855))
+- Remove LegacyESVersion.V_7_6_ and V_7_7_ Constants ([#4837](https://github.com/opensearch-project/OpenSearch/pull/4837))
+- Remove LegacyESVersion.V_7_10_ Constants ([#5018](https://github.com/opensearch-project/OpenSearch/pull/5018))
+- Remove Version.V_1_ Constants ([#5021](https://github.com/opensearch-project/OpenSearch/pull/5021))
+- Remove custom Map, List and Set collection classes ([#6871](https://github.com/opensearch-project/OpenSearch/pull/6871))
+
+### Fixed
+- Fix 'org.apache.hc.core5.http.ParseException: Invalid protocol version' under JDK 16+ ([#4827](https://github.com/opensearch-project/OpenSearch/pull/4827))
+- Fix compression support for h2c protocol ([#4944](https://github.com/opensearch-project/OpenSearch/pull/4944))
+- Don't over-allocate in HeapBufferedAsyncEntityConsumer in order to consume the response ([#9993](https://github.com/opensearch-project/OpenSearch/pull/9993))
+
+### Security
+
+[Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD
diff --git a/CHANGELOG.md b/CHANGELOG.md
index af20332c61146..145e29b36e84b 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,116 +3,35 @@ All notable changes to this project are documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). See the [CONTRIBUTING guide](./CONTRIBUTING.md#Changelog) for instructions on how to add changelog entries.
 
-## [Unreleased 3.0]
-### Added
-- Support for HTTP/2 (server-side) ([#3847](https://github.com/opensearch-project/OpenSearch/pull/3847))
-- Add getter for path field in NestedQueryBuilder ([#4636](https://github.com/opensearch-project/OpenSearch/pull/4636))
-- Allow mmap to use new JDK-19 preview APIs in Apache Lucene 9.4+ ([#5151](https://github.com/opensearch-project/OpenSearch/pull/5151))
-- Add events correlation engine plugin ([#6854](https://github.com/opensearch-project/OpenSearch/issues/6854))
-- Implement on behalf of token passing for extensions ([#8679](https://github.com/opensearch-project/OpenSearch/pull/8679), [#10664](https://github.com/opensearch-project/OpenSearch/pull/10664))
-- Provide service accounts tokens to extensions ([#9618](https://github.com/opensearch-project/OpenSearch/pull/9618))
-- GHA to verify checklist items completion in PR descriptions ([#10800](https://github.com/opensearch-project/OpenSearch/pull/10800))
-- Allow to pass the list settings through environment variables (like [], ["a", "b", "c"], ...) ([#10625](https://github.com/opensearch-project/OpenSearch/pull/10625))
-- [S3 Repository] Add setting to control connection count for sync client ([#12028](https://github.com/opensearch-project/OpenSearch/pull/12028))
-- Views, simplify data access and manipulation by providing a virtual layer over one or more indices ([#11957](https://github.com/opensearch-project/OpenSearch/pull/11957))
-- Add Remote Store Migration Experimental flag and allow mixed mode clusters under same ([#11986](https://github.com/opensearch-project/OpenSearch/pull/11986))
-- Remote reindex: Add support for configurable retry mechanism ([#12561](https://github.com/opensearch-project/OpenSearch/pull/12561))
-- [Admission Control] Integrate IO Usage Tracker to the Resource Usage Collector Service and Emit IO Usage Stats ([#11880](https://github.com/opensearch-project/OpenSearch/pull/11880))
-- Tracing for deep search path ([#12103](https://github.com/opensearch-project/OpenSearch/pull/12103))
-
-### Dependencies
-- Bump `log4j-core` from 2.18.0 to 2.19.0
-- Bump `forbiddenapis` from 3.3 to 3.4
-- Bump `avro` from 1.11.1 to 1.11.2
-- Bump `woodstox-core` from 6.3.0 to 6.3.1
-- Bump `xmlbeans` from 5.1.0 to 5.1.1 ([#4354](https://github.com/opensearch-project/OpenSearch/pull/4354))
-- Bump `reactive-streams` from 1.0.3 to 1.0.4 ([#4488](https://github.com/opensearch-project/OpenSearch/pull/4488))
-- Bump `jempbox` from 1.8.16 to 1.8.17 ([#4550](https://github.com/opensearch-project/OpenSearch/pull/4550))
-- Update to Gradle 7.6 and JDK-19 ([#4973](https://github.com/opensearch-project/OpenSearch/pull/4973))
-- Update Apache Lucene to 9.5.0-snapshot-d5cef1c ([#5570](https://github.com/opensearch-project/OpenSearch/pull/5570))
-- Bump `maven-model` from 3.6.2 to 3.8.6 ([#5599](https://github.com/opensearch-project/OpenSearch/pull/5599))
-- Bump `maxmind-db` from 2.1.0 to 3.0.0 ([#5601](https://github.com/opensearch-project/OpenSearch/pull/5601))
-- Bump `wiremock-jre8-standalone` from 2.33.2 to 2.35.0
-- Bump `gson` from 2.10 to 2.10.1
-- Bump `com.google.code.gson:gson` from 2.10 to 2.10.1
-- Bump `com.maxmind.geoip2:geoip2` from 4.0.0 to 4.0.1
-- Bump `com.avast.gradle:gradle-docker-compose-plugin` from 0.16.11 to 0.16.12
-- Bump `org.apache.commons:commons-configuration2` from 2.8.0 to 2.9.0
-- Bump `com.netflix.nebula:nebula-publishing-plugin` from 19.2.0 to 20.3.0
-- Bump `io.opencensus:opencensus-api` from 0.18.0 to 0.31.1 ([#7291](https://github.com/opensearch-project/OpenSearch/pull/7291))
-- OpenJDK Update (April 2023 Patch releases) ([#7344](https://github.com/opensearch-project/OpenSearch/pull/7344)
-- Bump `com.google.http-client:google-http-client:1.43.2` from 1.42.0 to 1.43.2 ([7928](https://github.com/opensearch-project/OpenSearch/pull/7928)))
-- Add Opentelemetry dependencies ([#7543](https://github.com/opensearch-project/OpenSearch/issues/7543))
-- Bump `org.bouncycastle:bcprov-jdk15on` to `org.bouncycastle:bcprov-jdk15to18` version 1.75 ([#8247](https://github.com/opensearch-project/OpenSearch/pull/8247))
-- Bump `org.bouncycastle:bcmail-jdk15on` to `org.bouncycastle:bcmail-jdk15to18` version 1.75 ([#8247](https://github.com/opensearch-project/OpenSearch/pull/8247))
-- Bump `org.bouncycastle:bcpkix-jdk15on` to `org.bouncycastle:bcpkix-jdk15to18` version 1.75 ([#8247](https://github.com/opensearch-project/OpenSearch/pull/8247))
-- Bump JNA version from 5.5 to 5.13 ([#9963](https://github.com/opensearch-project/OpenSearch/pull/9963))
-- Bump `org.eclipse.jgit` from 6.5.0 to 6.7.0 ([#10147](https://github.com/opensearch-project/OpenSearch/pull/10147))
-- Bump OpenTelemetry from 1.30.1 to 1.31.0 ([#10617](https://github.com/opensearch-project/OpenSearch/pull/10617))
-- Bump OpenTelemetry from 1.31.0 to 1.32.0 and OpenTelemetry Semconv from 1.21.0-alpha to 1.23.1-alpha ([#11305](https://github.com/opensearch-project/OpenSearch/pull/11305))
-- Bump `org.bouncycastle:bcprov-jdk15to18` to `org.bouncycastle:bcprov-jdk18on` version 1.77 ([#12317](https://github.com/opensearch-project/OpenSearch/pull/12317))
-- Bump `org.bouncycastle:bcmail-jdk15to18` to `org.bouncycastle:bcmail-jdk18on` version 1.77 ([#12317](https://github.com/opensearch-project/OpenSearch/pull/12317))
-- Bump `org.bouncycastle:bcpkix-jdk15to18` to `org.bouncycastle:bcpkix-jdk18on` version 1.77 ([#12317](https://github.com/opensearch-project/OpenSearch/pull/12317))
-- Bump Jackson version from 2.16.1 to 2.16.2 ([#12611](https://github.com/opensearch-project/OpenSearch/pull/12611))
-- Bump `aws-sdk-java` from 2.20.55 to 2.20.86 ([#12251](https://github.com/opensearch-project/OpenSearch/pull/12251))
-
-### Changed
-- [CCR] Add getHistoryOperationsFromTranslog method to fetch the history snapshot from translogs ([#3948](https://github.com/opensearch-project/OpenSearch/pull/3948))
-- Relax visibility of the HTTP_CHANNEL_KEY and HTTP_SERVER_CHANNEL_KEY to make it possible for the plugins to access associated Netty4HttpChannel / Netty4HttpServerChannel instance ([#4638](https://github.com/opensearch-project/OpenSearch/pull/4638))
-- Migrate client transports to Apache HttpClient / Core 5.x ([#4459](https://github.com/opensearch-project/OpenSearch/pull/4459))
-- Change http code on create index API with bad input raising NotXContentException from 500 to 400 ([#4773](https://github.com/opensearch-project/OpenSearch/pull/4773))
-- Improve summary error message for invalid setting updates ([#4792](https://github.com/opensearch-project/OpenSearch/pull/4792))
-- Return 409 Conflict HTTP status instead of 503 on failure to concurrently execute snapshots ([#8986](https://github.com/opensearch-project/OpenSearch/pull/5855))
-- Add task completion count in search backpressure stats API ([#10028](https://github.com/opensearch-project/OpenSearch/pull/10028/))
-- Deprecate CamelCase `PathHierarchy` tokenizer name in favor to lowercase `path_hierarchy` ([#10894](https://github.com/opensearch-project/OpenSearch/pull/10894))
-- Switched to more reliable OpenSearch Lucene snapshot location([#11728](https://github.com/opensearch-project/OpenSearch/pull/11728))
-- Breaking change: Do not request "search_pipelines" metrics by default in NodesInfoRequest ([#12497](https://github.com/opensearch-project/OpenSearch/pull/12497))
-
-### Deprecated
-
-### Removed
-- Remove deprecated code to add node name into log pattern of log4j property file ([#4568](https://github.com/opensearch-project/OpenSearch/pull/4568))
-- Unused object and import within TransportClusterAllocationExplainAction ([#4639](https://github.com/opensearch-project/OpenSearch/pull/4639))
-- Remove LegacyESVersion.V_7_0_* and V_7_1_* Constants ([#2768](https://https://github.com/opensearch-project/OpenSearch/pull/2768))
-- Remove LegacyESVersion.V_7_2_ and V_7_3_ Constants ([#4702](https://github.com/opensearch-project/OpenSearch/pull/4702))
-- Always auto release the flood stage block ([#4703](https://github.com/opensearch-project/OpenSearch/pull/4703))
-- Remove LegacyESVersion.V_7_4_ and V_7_5_ Constants ([#4704](https://github.com/opensearch-project/OpenSearch/pull/4704))
-- Remove Legacy Version support from Snapshot/Restore Service ([#4728](https://github.com/opensearch-project/OpenSearch/pull/4728))
-- Remove deprecated serialization logic from pipeline aggs ([#4847](https://github.com/opensearch-project/OpenSearch/pull/4847))
-- Remove unused private methods ([#4926](https://github.com/opensearch-project/OpenSearch/pull/4926))
-- Remove LegacyESVersion.V_7_8_ and V_7_9_ Constants ([#4855](https://github.com/opensearch-project/OpenSearch/pull/4855))
-- Remove LegacyESVersion.V_7_6_ and V_7_7_ Constants ([#4837](https://github.com/opensearch-project/OpenSearch/pull/4837))
-- Remove LegacyESVersion.V_7_10_ Constants ([#5018](https://github.com/opensearch-project/OpenSearch/pull/5018))
-- Remove Version.V_1_ Constants ([#5021](https://github.com/opensearch-project/OpenSearch/pull/5021))
-- Remove custom Map, List and Set collection classes ([#6871](https://github.com/opensearch-project/OpenSearch/pull/6871))
-
-### Fixed
-- Fix 'org.apache.hc.core5.http.ParseException: Invalid protocol version' under JDK 16+ ([#4827](https://github.com/opensearch-project/OpenSearch/pull/4827))
-- Fix compression support for h2c protocol ([#4944](https://github.com/opensearch-project/OpenSearch/pull/4944))
-- Don't over-allocate in HeapBufferedAsyncEntityConsumer in order to consume the response ([#9993](https://github.com/opensearch-project/OpenSearch/pull/9993))
-- Update supported version for max_shard_size parameter in Shrink API ([#11439](https://github.com/opensearch-project/OpenSearch/pull/11439))
-- Fix typo in API annotation check message ([11836](https://github.com/opensearch-project/OpenSearch/pull/11836))
-- Update supported version for must_exist parameter in update aliases API ([#11872](https://github.com/opensearch-project/OpenSearch/pull/11872))
-- [Bug] Check phase name before SearchRequestOperationsListener onPhaseStart ([#12035](https://github.com/opensearch-project/OpenSearch/pull/12035))
-- Fix Span operation names generated from RestActions ([#12005](https://github.com/opensearch-project/OpenSearch/pull/12005))
-- Fix error in RemoteSegmentStoreDirectory when debug logging is enabled ([#12328](https://github.com/opensearch-project/OpenSearch/pull/12328))
-
-### Security
-
 ## [Unreleased 2.x]
 ### Added
+- Add latency metrics for instrumenting critical clusterManager code paths ([#12333](https://github.com/opensearch-project/OpenSearch/pull/12333))
+- Add support for Azure Managed Identity in repository-azure ([#12423](https://github.com/opensearch-project/OpenSearch/issues/12423))
+- Add useCompoundFile index setting ([#13478](https://github.com/opensearch-project/OpenSearch/pull/13478))
+- Make outbound side of transport protocol dependent ([#13293](https://github.com/opensearch-project/OpenSearch/pull/13293))
 
 ### Dependencies
+- Bump `com.github.spullara.mustache.java:compiler` from 0.9.10 to 0.9.13 ([#13329](https://github.com/opensearch-project/OpenSearch/pull/13329), [#13559](https://github.com/opensearch-project/OpenSearch/pull/13559))
+- Bump `org.gradle.test-retry` from 1.5.8 to 1.5.9 ([#13442](https://github.com/opensearch-project/OpenSearch/pull/13442))
+- Bump `org.apache.commons:commons-text` from 1.11.0 to 1.12.0 ([#13557](https://github.com/opensearch-project/OpenSearch/pull/13557))
+- Bump `org.hdrhistogram:HdrHistogram` from 2.1.12 to 2.2.1 ([#13556](https://github.com/opensearch-project/OpenSearch/pull/13556))
+- Bump `com.gradle.enterprise` from 3.17.2 to 3.17.3 ([#13641](https://github.com/opensearch-project/OpenSearch/pull/13641))
+- Bump `org.apache.hadoop:hadoop-minicluster` from 3.3.6 to 3.4.0 ([#13642](https://github.com/opensearch-project/OpenSearch/pull/13642))
 
 ### Changed
+- Add ability for Boolean and date field queries to run when only doc_values are enabled ([#11650](https://github.com/opensearch-project/OpenSearch/pull/11650))
+- Refactor implementations of query phase searcher, allow QueryCollectorContext to have zero collectors ([#13481](https://github.com/opensearch-project/OpenSearch/pull/13481))
 
 ### Deprecated
 
 ### Removed
+- Remove handling of index.mapper.dynamic in AutoCreateIndex([#13067](https://github.com/opensearch-project/OpenSearch/pull/13067))
 
 ### Fixed
+- Fix negative RequestStats metric issue ([#13553](https://github.com/opensearch-project/OpenSearch/pull/13553))
+- Fix get field mapping API returns 404 error in mixed cluster with multiple versions ([#13624](https://github.com/opensearch-project/OpenSearch/pull/13624))
+- Allow clearing `remote_store.compatibility_mode` setting ([#13646](https://github.com/opensearch-project/OpenSearch/pull/13646))
 
 ### Security
 
-[Unreleased 3.0]: https://github.com/opensearch-project/OpenSearch/compare/2.x...HEAD
-[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.12...2.x
+[Unreleased 2.x]: https://github.com/opensearch-project/OpenSearch/compare/2.13...2.x
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 4a1162cf2558b..0ec0abe535dd0 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -1,18 +1,19 @@
 - [Contributing to OpenSearch](#contributing-to-opensearch)
-  - [First Things First](#first-things-first)
-  - [Ways to Contribute](#ways-to-contribute)
+- [First Things First](#first-things-first)
+- [Ways to Contribute](#ways-to-contribute)
     - [Bug Reports](#bug-reports)
     - [Feature Requests](#feature-requests)
     - [Documentation Changes](#documentation-changes)
     - [Contributing Code](#contributing-code)
-  - [Developer Certificate of Origin](#developer-certificate-of-origin)
-  - [Changelog](#changelog)
-  - [Review Process](#review-process)
-  - [Troubleshooting Failing Builds](#troubleshooting-failing-builds)
+- [Developer Certificate of Origin](#developer-certificate-of-origin)
+- [Changelog](#changelog)
+- [Review Process](#review-process)
+    - [Tips for Success](#tips)
+- [Troubleshooting Failing Builds](#troubleshooting-failing-builds)
 
 # Contributing to OpenSearch
 
-OpenSearch is a community project that is built and maintained by people just like **you**. We're glad you're interested in helping out. There are several different ways you can do it, but before we talk about that, let's talk about how to get started.
+OpenSearch is a community project built and maintained by people just like **you**. We're glad you're interested in helping out. There are several different ways you can do it, but before we talk about that, let's talk about how to get started.
 
 ## First Things First
 
@@ -30,9 +31,9 @@ Ugh! Bugs!
 
 A bug is when software behaves in a way that you didn't expect and the developer didn't intend. To help us understand what's going on, we first want to make sure you're working from the latest version. Please make sure you're testing against the [latest version](https://github.com/opensearch-project/OpenSearch).
 
-Once you've confirmed that the bug still exists in the latest version, you'll want to check to make sure it's not something we already know about on the [open issues GitHub page](https://github.com/opensearch-project/OpenSearch/issues).
+Once you've confirmed that the bug still exists in the latest version, you'll want to check the bug is not something we already know about. A good way to figure this out is to search for your bug on the [open issues GitHub page](https://github.com/opensearch-project/OpenSearch/issues).
 
-If you've upgraded to the latest version and you can't find it in our open issues list, then you'll need to tell us how to reproduce it. To make the behavior as clear as possible, please provided your steps as `curl` commands which we can copy and paste into a terminal to run it locally, for example:
+If you've upgraded to the latest version and you can't find it in our open issues list, then you'll need to tell us how to reproduce it. To make the behavior as clear as possible, please provide your steps as `curl` commands which we can copy and paste into a terminal to run it locally, for example:
 
 ```sh
 # delete the index
@@ -47,11 +48,11 @@ curl -x PUT localhost:9200/test/test/1 -d '{
 curl ....
 ```
 
-Provide as much information as you can. You may think that the problem lies with your query, when actually it depends on how your data is indexed. The easier it is for us to recreate your problem, the faster it is likely to be fixed.
+Provide as much information as you can. You may think that the problem lies with your query, when actually it depends on how your data is indexed. The easier it is for us to recreate your problem, the faster it is likely to be fixed. It is generally always helpful to provide the basic details of your cluster configuration alongside your reproduction steps.
 
 ### Feature Requests
 
-If you've thought of a way that OpenSearch could be better, we want to hear about it. We track feature requests using GitHub, so please feel free to open an issue which describes the feature you would like to see, why you need it, and how it should work.
+If you've thought of a way that OpenSearch could be better, we want to hear about it. We track feature requests using GitHub, so please feel free to open an issue which describes the feature you would like to see, why you need it, and how it should work. After opening an issue, the fastest way to see your change made is to open a pull request following the requested changes you detailed in your issue. You can learn more about opening a pull request in the [contributing code section](#contributing-code).
 
 ### Documentation Changes
 
@@ -146,12 +147,12 @@ Adding in the change is two step process:
 2. Update the entry for your change in [`CHANGELOG.md`](CHANGELOG.md) and make sure that you reference the pull request there.
 
 ### Where should I put my CHANGELOG entry?
-Please review the [branching strategy](https://github.com/opensearch-project/.github/blob/main/RELEASING.md#opensearch-branching) document. The changelog on the `main` branch will contain sections for the _next major_ and _next minor_ releases. Your entry should go into the section it is intended to be released in. In practice, most changes to `main` will be backported to the next minor release so most entries will likely be in that section.
+Please review the [branching strategy](https://github.com/opensearch-project/.github/blob/main/RELEASING.md#opensearch-branching) document. The changelog on the `main` branch will contain **two files**: `CHANGELOG.md` which corresponds to unreleased changes intended for the _next minor_ release and `CHANGELOG-3.0.md` which correspond to unreleased changes intended for the _next major_ release. Your entry should go into file corresponding to the version it is intended to be released in. In practice, most changes to `main` will be backported to the next minor release so most entries will be in the `CHANGELOG.md` file.
 
 The following examples assume the _next major_ release on main is 3.0, then _next minor_ release is 2.5, and the _current_ release is 2.4.
 
-- **Add a new feature to release in next minor:** Add a changelog entry to `[Unreleased 2.x]` on main, then backport to 2.x (including the changelog entry).
-- **Introduce a breaking API change to release in next major:** Add a changelog entry to `[Unreleased 3.0]` on main, do not backport.
+- **Add a new feature to release in next minor:** Add a changelog entry to `[Unreleased 2.x]` in CHANGELOG.md on main, then backport to 2.x (including the changelog entry).
+- **Introduce a breaking API change to release in next major:** Add a changelog entry to `[Unreleased 3.0]` to CHANGELOG-3.0.md on main, do not backport.
 - **Upgrade a dependency to fix a CVE:** Add a changelog entry to `[Unreleased 2.x]` on main, then backport to 2.x (including the changelog entry), then backport to 2.4 and ensure the changelog entry is added to `[Unreleased 2.4.1]`.
 
 ## Review Process
@@ -164,13 +165,29 @@ If we accept the PR, a [maintainer](MAINTAINERS.md) will merge your change and u
 
 If we reject the PR, we will close the pull request with a comment explaining why. This decision isn't always final: if you feel we have misunderstood your intended change or otherwise think that we should reconsider then please continue the conversation with a comment on the PR and we'll do our best to address any further points you raise.
 
+### Tips for Success (#tips)
+
+We have a lot of mechanisms to help expedite towards an accepted PR. Here are some tips for success:
+1. *Minimize BWC guarantees*: The first PR review cycle heavily focuses on the public facing APIs. This is what we have to "guarantee" as non-breaking for [bwc across major versions](./DEVELOPER_GUIDE.md#backwards-compatibility).
+2. *Do not copy non-compliant code*: Ensure that code is APLv2 compatible. This means that you have not copied any code from other sources unless that code is also APLv2 compatible.
+3. *Utilize feature flags*: Features that are safeguarded behind feature flags are more likely to be merged and backported, as they come with an additional layer of protection. Refer to this [example PR](https://github.com/opensearch-project/OpenSearch/pull/4959) for implementation details.
+4. *Use appropriate Java tags*:
+    - `@opensearch.internal`: Marks internal classes subject to rapid changes.
+    - `@opensearch.api`: Marks public-facing API classes with backward compatibility guarantees.
+    - `@opensearch.experimental`: Indicates rapidly changing [experimental code](./DEVELOPER_GUIDE.md#experimental-development).
+5. *Employ sandbox for significant core changes*: Any new features or enhancements that make changes to core classes (e.g., search phases, codecs, or specialized lucene APIs) are more likely to. be merged if they are sandboxed. This can only be enabled on the java CLI (`-Dsandbox.enabled=true`).
+6. *Micro-benchmark critical path*: This is a lesser known mechanism, but if you have critical path changes you're afraid will impact performance (the changes touch the garbage collector, heap, direct memory, or CPU) then including a [microbenchmark](https://github.com/opensearch-project/OpenSearch/tree/main/benchmarks) with your PR (and jfr or flamegraph results in the description) is a *GREAT IDEA* and will help expedite the review process.
+7. *Test rigorously*: Ensure thorough testing ([OpenSearchTestCase](./test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java) for unit tests, [OpenSearchIntegTestCase](./test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java) for integration & cluster tests, [OpenSearchRestTestCase](./test/framework/src/main/java/org/opensearch/test/rest/OpenSearchRestTestCase.java) for testing REST endpoint interfaces, and yaml tests with [ClientYamlTestSuiteIT](./rest-api-spec/src/yamlRestTest/java/org/opensearch/test/rest/ClientYamlTestSuiteIT.java) for REST integration tests)
+
+In general, adding more guardrails to your changes increases the likelihood of swift PR acceptance. We can always relax these guard rails in smaller followup PRs. Reverting a GA feature is much more difficult. Check out the [DEVELOPER_GUIDE](./DEVELOPER_GUIDE.md#submitting-changes) for more useful tips.
+
 ## Troubleshooting Failing Builds
 
-The OpenSearch testing framework offers many capabilities but exhibits significant complexity (it does lot of randomization internally to cover as many edge cases and variations as possible). Unfortunately, this posses a challenge by making it harder to discover important issues/bugs in straightforward way and may lead to so called flaky tests - the tests which flip randomly from success to failure without any code changes. 
+The OpenSearch testing framework offers many capabilities but exhibits significant complexity (it does lot of randomization internally to cover as many edge cases and variations as possible). Unfortunately, this posses a challenge by making it harder to discover important issues/bugs in straightforward way and may lead to so called flaky tests - the tests which flip randomly from success to failure without any code changes.
 
 If your pull request reports a failing test(s) on one of the checks, please:
- - look if there is an existing [issue](https://github.com/opensearch-project/OpenSearch/issues) reported for the test in question 
- - if not, please make sure this is not caused by your changes, run the failing test(s) locally for some time
- - if you are sure the failure is not related, please open a new [bug](https://github.com/opensearch-project/OpenSearch/issues/new?assignees=&labels=bug%2C+untriaged&projects=&template=bug_template.md&title=%5BBUG%5D) with `flaky-test` label
- - add a comment referencing the issue(s) or bug report(s) to your pull request explaining the failing build(s)
- - as a bonus point, try to contribute by fixing the flaky test(s)
+- look if there is an existing [issue](https://github.com/opensearch-project/OpenSearch/issues) reported for the test in question
+- if not, please make sure this is not caused by your changes, run the failing test(s) locally for some time
+- if you are sure the failure is not related, please open a new [bug](https://github.com/opensearch-project/OpenSearch/issues/new?assignees=&labels=bug%2C+untriaged&projects=&template=bug_template.md&title=%5BBUG%5D) with `flaky-test` label
+- add a comment referencing the issue(s) or bug report(s) to your pull request explaining the failing build(s)
+- as a bonus point, try to contribute by fixing the flaky test(s)
diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md
index f0851fc58d444..92ef71b92da7e 100644
--- a/DEVELOPER_GUIDE.md
+++ b/DEVELOPER_GUIDE.md
@@ -57,6 +57,7 @@
       - [Developer API](#developer-api)
       - [User API](#user-api)
       - [Experimental Development](#experimental-development)
+      - [API Compatibility Checks](#api-compatibility-checks)
     - [Backports](#backports)
     - [LineLint](#linelint)
     - [Lucene Snapshots](#lucene-snapshots)
@@ -607,6 +608,20 @@ a LTS feature but with additional guard rails and communication mechanisms to si
 release, or be removed altogether. Any Developer or User APIs implemented along with the experimental feature should be marked with `@ExperimentalApi` (or documented as
 `@opensearch.experimental`) annotation to signal the implementation is not subject to LTS and does not follow backwards compatibility guidelines.
 
+#### API Compatibility Checks
+
+The compatibility checks for public APIs are performed using [japicmp](https://siom79.github.io/japicmp/) and are available as separate Gradle tasks (those are run on demand at the moment):
+
+```
+./gradlew japicmp
+```
+
+By default, the API compatibility checks are run against the latest released version of the OpenSearch, however the target version to compare to could be provided using system property during the build, fe.:
+
+```
+./gradlew japicmp  -Djapicmp.compare.version=2.14.0-SNAPSHOT
+```
+
 ### Backports
 
 The Github workflow in [`backport.yml`](.github/workflows/backport.yml) creates backport PRs automatically when the original PR with an appropriate label `backport <backport-branch-name>` is merged to main with the backport workflow run successfully on the PR. For example, if a PR on main needs to be backported to `1.x` branch, add a label `backport 1.x` to the PR and make sure the backport workflow runs on the PR along with other checks. Once this PR is merged to main, the workflow will create a backport PR to the `1.x` branch.
diff --git a/MAINTAINERS.md b/MAINTAINERS.md
index 5535c2fa26eae..cce92167473b6 100644
--- a/MAINTAINERS.md
+++ b/MAINTAINERS.md
@@ -5,8 +5,7 @@ This document contains a list of maintainers in this repo. See [opensearch-proje
 ## Current Maintainers
 
 | Maintainer               | GitHub ID                                               | Affiliation |
-|--------------------------| ------------------------------------------------------- | ----------- |
-| Abbas Hussain            | [abbashus](https://github.com/abbashus)                 | Meta        |
+| ------------------------ | ------------------------------------------------------- | ----------- |
 | Anas Alkouz              | [anasalkouz](https://github.com/anasalkouz)             | Amazon      |
 | Andrew Ross              | [andrross](https://github.com/andrross)                 | Amazon      |
 | Andriy Redko             | [reta](https://github.com/reta)                         | Aiven       |
@@ -15,16 +14,13 @@ This document contains a list of maintainers in this repo. See [opensearch-proje
 | Dan Widdis               | [dbwiddis](https://github.com/dbwiddis)                 | Amazon      |
 | Daniel "dB." Doubrovkine | [dblock](https://github.com/dblock)                     | Amazon      |
 | Gaurav Bafna             | [gbbafna](https://github.com/gbbafna)                   | Amazon      |
-| Himanshu Setia           | [setiah](https://github.com/setiah)                     | Amazon      |
 | Kunal Kotwani            | [kotwanikunal](https://github.com/kotwanikunal)         | Amazon      |
 | Marc Handalian           | [mch2](https://github.com/mch2)                         | Amazon      |
 | Michael Froh             | [msfroh](https://github.com/msfroh)                     | Amazon      |
 | Nick Knize               | [nknize](https://github.com/nknize)                     | Amazon      |
 | Owais Kazi               | [owaiskazi19](https://github.com/owaiskazi19)           | Amazon      |
 | Peter Nied               | [peternied](https://github.com/peternied)               | Amazon      |
-| Rabi Panda               | [adnapibar](https://github.com/adnapibar)               | Independent |
 | Rishikesh Pasham         | [Rishikesh1159](https://github.com/Rishikesh1159)       | Amazon      |
-| Ryan Bogan               | [ryanbogan](https://github.com/ryanbogan)               | Amazon      |
 | Sachin Kale              | [sachinpkale](https://github.com/sachinpkale)           | Amazon      |
 | Sarat Vemulapalli        | [saratvemulapalli](https://github.com/saratvemulapalli) | Amazon      |
 | Shweta Thareja           | [shwetathareja](https://github.com/shwetathareja)       | Amazon      |
@@ -35,8 +31,12 @@ This document contains a list of maintainers in this repo. See [opensearch-proje
 
 ## Emeritus
 
-| Maintainer              | GitHub ID                                   | Affiliation |
-|-------------------------|---------------------------------------------|-------------|
-| Megha Sai Kavikondala   | [meghasaik](https://github.com/meghasaik)   | Amazon      |
-| Xue Zhou                | [xuezhou25](https://github.com/xuezhou25)   | Amazon      |
-| Kartik Ganesh           | [kartg](https://github.com/kartg)           | Amazon      |
+| Maintainer            | GitHub ID                                 | Affiliation |
+| --------------------- | ----------------------------------------- | ----------- |
+| Megha Sai Kavikondala | [meghasaik](https://github.com/meghasaik) | Amazon      |
+| Xue Zhou              | [xuezhou25](https://github.com/xuezhou25) | Amazon      |
+| Kartik Ganesh         | [kartg](https://github.com/kartg)         | Amazon      |
+| Abbas Hussain         | [abbashus](https://github.com/abbashus)   | Meta        |
+| Himanshu Setia        | [setiah](https://github.com/setiah)       | Amazon      |
+| Ryan Bogan            | [ryanbogan](https://github.com/ryanbogan) | Amazon      |
+| Rabi Panda            | [adnapibar](https://github.com/adnapibar) | Independent |
diff --git a/README.md b/README.md
index b5fc45509b002..aff18018960d5 100644
--- a/README.md
+++ b/README.md
@@ -7,8 +7,8 @@
 [![Security Vulnerabilities](https://img.shields.io/github/issues/opensearch-project/OpenSearch/security%20vulnerability?labelColor=red)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"security%20vulnerability")
 [![Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch)](https://github.com/opensearch-project/OpenSearch/issues)
 [![Open Pull Requests](https://img.shields.io/github/issues-pr/opensearch-project/OpenSearch)](https://github.com/opensearch-project/OpenSearch/pulls)
-[![2.10 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.10.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.10.0")
-[![3.0 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v3.0.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v3.0.0")
+[![2.14.0 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v2.14.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v2.14.0")
+[![3.0.0 Open Issues](https://img.shields.io/github/issues/opensearch-project/OpenSearch/v3.0.0)](https://github.com/opensearch-project/OpenSearch/issues?q=is%3Aissue+is%3Aopen+label%3A"v3.0.0")
 [![GHA gradle check](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml)
 [![GHA validate pull request](https://github.com/opensearch-project/OpenSearch/actions/workflows/wrapper.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/wrapper.yml)
 [![GHA precommit](https://github.com/opensearch-project/OpenSearch/actions/workflows/precommit.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/precommit.yml)
@@ -45,7 +45,7 @@
 This project has adopted the [Amazon Open Source Code of Conduct](CODE_OF_CONDUCT.md). For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq), or contact [opensource-codeofconduct@amazon.com](mailto:opensource-codeofconduct@amazon.com) with any additional questions or comments.
 
 ## Security
-If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/) or directly via email to aws-security@amazon.com. Please do **not** create a public GitHub issue.
+If you discover a potential security issue in this project we ask that you notify OpenSearch Security directly via email to security@opensearch.org. Please do **not** create a public GitHub issue.
 
 ## License
 
diff --git a/SECURITY.md b/SECURITY.md
index b86292104335f..be4ac7463864a 100644
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -1,3 +1,3 @@
 ## Reporting a Vulnerability
 
-If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/) or directly via email to aws-security@amazon.com. Please do **not** create a public GitHub issue.
+If you discover a potential security issue in this project we ask that you notify OpenSearch Security directly via email to security@opensearch.org. Please do **not** create a public GitHub issue.
diff --git a/TESTING.md b/TESTING.md
index 1c91d60840d61..80fc2412d736b 100644
--- a/TESTING.md
+++ b/TESTING.md
@@ -84,6 +84,7 @@ This will instruct all JVMs (including any that run cli tools such as creating t
 -   In order to remotely attach a debugger to the process: `--debug-jvm`
 -   In order to set a different keystore password: `--keystore-password yourpassword`
 -   In order to set an OpenSearch setting, provide a setting with the following prefix: `-Dtests.opensearch.`
+-   In order to enable stack trace of the MockSpanData during testing, add: `-Dtests.telemetry.span.stack_traces=true` (Storing stack traces alongside span data can be useful for comprehensive debugging and performance optimization during testing, as it provides insights into the exact code paths and execution sequences, facilitating efficient issue identification and resolution. Note: Enabling this might lead to OOM issues while running ITs)
 
 ## Test case filtering
 
@@ -412,8 +413,8 @@ Say you need to make a change to `main` and have a BWC layer in `5.x`. You will
 You may want to run BWC tests for a secure OpenSearch cluster. In order to do this, you will need to follow a few additional steps:
 
 1. Clone the OpenSearch Security repository from https://github.com/opensearch-project/security.
-2. Get both the old version of the Security plugin (the version you wish to come from) and the new version of the Security plugin (the version you wish to go to). This can be done either by fetching the maven artifact with a command like `wget https://repo1.maven.org/maven2/org/opensearch/plugin/opensearch-security/<TARGET_VERSION>.0/opensearch-security-<TARGET_VERSION>.0.zip` or by running `./gradlew assemble` from the base of the Security repository. 
-3. Move both of the Security artifacts into new directories at the path `/security/bwc-test/src/test/resources/<TARGET_VERSION>.0`. You should end up with two different directories in `/security/bwc-test/src/test/resources/`, one named the old version and one the new version. 
+2. Get both the old version of the Security plugin (the version you wish to come from) and the new version of the Security plugin (the version you wish to go to). This can be done either by fetching the maven artifact with a command like `wget https://repo1.maven.org/maven2/org/opensearch/plugin/opensearch-security/<TARGET_VERSION>.0/opensearch-security-<TARGET_VERSION>.0.zip` or by running `./gradlew assemble` from the base of the Security repository.
+3. Move both of the Security artifacts into new directories at the path `/security/bwc-test/src/test/resources/<TARGET_VERSION>.0`. You should end up with two different directories in `/security/bwc-test/src/test/resources/`, one named the old version and one the new version.
 4. Run the following command from the base of the Security repository:
 
 ```
@@ -428,7 +429,7 @@ You may want to run BWC tests for a secure OpenSearch cluster. In order to do th
 
 `-Dtests.security.manager=false` handles access issues when attempting to read the certificates from the file system.
 `-Dtests.opensearch.http.protocol=https` tells the wait for cluster startup task to do the right thing.
-`-PcustomDistributionUrl=...` uses a custom build of the distribution of OpenSearch. This is unnecessary when running against standard/unmodified OpenSearch core distributions. 
+`-PcustomDistributionUrl=...` uses a custom build of the distribution of OpenSearch. This is unnecessary when running against standard/unmodified OpenSearch core distributions.
 
 ### Skip fetching latest
 
diff --git a/TRIAGING.md b/TRIAGING.md
index 3917f1e1442b9..bb04d49a66c54 100644
--- a/TRIAGING.md
+++ b/TRIAGING.md
@@ -14,9 +14,9 @@ Each meeting we seek to address all new issues. However, should we run out of ti
 
 ### How do I join the Triage meeting?
 
-Meetings are hosted regularly at 10:00a - 10:55a Central Time every Wednesday and can be joined via [Chime](https://aws.amazon.com/chime/), with this [meeting link](https://chime.aws/1988437365).
+Meetings are hosted regularly at 10:00a - 10:55a Central Time every Wednesday and can be joined via [Zoom](https://zoom.us/download), with this [meeting link](https://us02web.zoom.us/j/86287450465) and passcode `805212`.
 
-After joining the Chime meeting, you can enable your video / voice to join the discussion.  If you do not have a webcam or microphone available, you can still join in via the text chat.
+After joining the Zoom meeting, you can enable your video / voice to join the discussion.  If you do not have a webcam or microphone available, you can still join in via the text chat.
 
 If you have an issue you'd like to bring forth please prepare a link to the issue so it can be presented and viewed by everyone in the meeting.
 
diff --git a/build.gradle b/build.gradle
index 2aac4a1e893e9..e92f396e006f5 100644
--- a/build.gradle
+++ b/build.gradle
@@ -55,7 +55,7 @@ plugins {
   id 'opensearch.docker-support'
   id 'opensearch.global-build-info'
   id "com.diffplug.spotless" version "6.25.0" apply false
-  id "org.gradle.test-retry" version "1.5.8" apply false
+  id "org.gradle.test-retry" version "1.5.9" apply false
   id "test-report-aggregation"
   id 'jacoco-report-aggregation'
 }
diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle
index 0562ecc6ee61b..c68cc0406d3a6 100644
--- a/buildSrc/build.gradle
+++ b/buildSrc/build.gradle
@@ -102,14 +102,14 @@ dependencies {
 
   api localGroovy()
 
-  api 'commons-codec:commons-codec:1.16.0'
-  api 'org.apache.commons:commons-compress:1.25.0'
+  api "commons-codec:commons-codec:${props.getProperty('commonscodec')}"
+  api "org.apache.commons:commons-compress:${props.getProperty('commonscompress')}"
   api 'org.apache.ant:ant:1.10.14'
   api 'com.netflix.nebula:gradle-extra-configurations-plugin:10.0.0'
   api 'com.netflix.nebula:nebula-publishing-plugin:21.0.0'
   api 'com.netflix.nebula:gradle-info-plugin:12.1.6'
   api 'org.apache.rat:apache-rat:0.15'
-  api 'commons-io:commons-io:2.15.1'
+  api "commons-io:commons-io:${props.getProperty('commonsio')}"
   api "net.java.dev.jna:jna:5.14.0"
   api 'com.github.johnrengelman:shadow:8.1.1'
   api 'org.jdom:jdom2:2.0.6.1'
diff --git a/buildSrc/src/main/java/org/opensearch/gradle/pluginzip/Publish.java b/buildSrc/src/main/java/org/opensearch/gradle/pluginzip/Publish.java
index 5d7e78589306f..599beb8649fcd 100644
--- a/buildSrc/src/main/java/org/opensearch/gradle/pluginzip/Publish.java
+++ b/buildSrc/src/main/java/org/opensearch/gradle/pluginzip/Publish.java
@@ -65,9 +65,6 @@ public void apply(Project project) {
                 addLocalMavenRepo(project);
                 addZipArtifact(project);
                 Task validatePluginZipPom = project.getTasks().findByName("validatePluginZipPom");
-                if (validatePluginZipPom != null) {
-                    validatePluginZipPom.dependsOn("generatePomFileForNebulaPublication");
-                }
 
                 // There are number of tasks prefixed by 'publishPluginZipPublication', f.e.:
                 // publishPluginZipPublicationToZipStagingRepository, publishPluginZipPublicationToMavenLocal
@@ -76,7 +73,11 @@ public void apply(Project project) {
                     .filter(t -> t.getName().startsWith("publishPluginZipPublicationTo"))
                     .collect(Collectors.toSet());
                 if (!publishPluginZipPublicationToTasks.isEmpty()) {
-                    publishPluginZipPublicationToTasks.forEach(t -> t.dependsOn("generatePomFileForNebulaPublication"));
+                    if (validatePluginZipPom != null) {
+                        publishPluginZipPublicationToTasks.forEach(t -> t.dependsOn(validatePluginZipPom));
+                    } else {
+                        publishPluginZipPublicationToTasks.forEach(t -> t.dependsOn("generatePomFileForNebulaPublication"));
+                    }
                 }
             } else {
                 project.getLogger()
diff --git a/buildSrc/src/main/java/org/opensearch/gradle/precommit/LicenseAnalyzer.java b/buildSrc/src/main/java/org/opensearch/gradle/precommit/LicenseAnalyzer.java
index 4c63516126566..c3acd12e5a1cf 100644
--- a/buildSrc/src/main/java/org/opensearch/gradle/precommit/LicenseAnalyzer.java
+++ b/buildSrc/src/main/java/org/opensearch/gradle/precommit/LicenseAnalyzer.java
@@ -145,7 +145,7 @@ public class LicenseAnalyzer {
                     + "AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n"
                     + "LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n"
                     + "OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\n"
-                    + "SOFTWARE\\.\n").replaceAll("\\s+", "\\\\s*"),
+                    + "SOFTWARE\\.?\n").replaceAll("\\s+", "\\\\s*"),
                 Pattern.DOTALL
             )
         ),
diff --git a/buildSrc/src/main/java/org/opensearch/gradle/precommit/PomValidationPrecommitPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/precommit/PomValidationPrecommitPlugin.java
index 0e7a357dd5d18..d3f173c9c02ea 100644
--- a/buildSrc/src/main/java/org/opensearch/gradle/precommit/PomValidationPrecommitPlugin.java
+++ b/buildSrc/src/main/java/org/opensearch/gradle/precommit/PomValidationPrecommitPlugin.java
@@ -53,12 +53,19 @@ public TaskProvider<? extends Task> createTask(Project project) {
             TaskProvider<PomValidationTask> validateTask = project.getTasks()
                 .register("validate" + publicationName + "Pom", PomValidationTask.class);
             validatePom.configure(t -> t.dependsOn(validateTask));
+            TaskProvider<GenerateMavenPom> generateMavenPom = project.getTasks()
+                .withType(GenerateMavenPom.class)
+                .named("generatePomFileFor" + publicationName + "Publication");
             validateTask.configure(task -> {
-                GenerateMavenPom generateMavenPom = project.getTasks()
-                    .withType(GenerateMavenPom.class)
-                    .getByName("generatePomFileFor" + publicationName + "Publication");
                 task.dependsOn(generateMavenPom);
-                task.getPomFile().fileValue(generateMavenPom.getDestination());
+                task.getPomFile().fileProvider(generateMavenPom.map(GenerateMavenPom::getDestination));
+                publishing.getPublications().all(publicationForPomGen -> {
+                    task.mustRunAfter(
+                        project.getTasks()
+                            .withType(GenerateMavenPom.class)
+                            .getByName("generatePomFileFor" + Util.capitalize(publicationForPomGen.getName()) + "Publication")
+                    );
+                });
             });
         });
 
diff --git a/buildSrc/src/main/java/org/opensearch/gradle/precommit/PomValidationTask.java b/buildSrc/src/main/java/org/opensearch/gradle/precommit/PomValidationTask.java
index aca882fbb6477..b76e0d6dd93cf 100644
--- a/buildSrc/src/main/java/org/opensearch/gradle/precommit/PomValidationTask.java
+++ b/buildSrc/src/main/java/org/opensearch/gradle/precommit/PomValidationTask.java
@@ -106,6 +106,7 @@ private <T> void validateNonNull(String element, T value, Runnable validator) {
 
     private void validateString(String element, String value) {
         validateNonNull(element, value, () -> validateNonEmpty(element, value, s -> s.trim().isEmpty()));
+        getLogger().info(element + " with value " + value + " is validated.");
     }
 
     private <T> void validateCollection(String element, Collection<T> value, Consumer<T> validator) {
diff --git a/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java b/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java
index bc44f81a81aff..b2b3e3003e572 100644
--- a/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java
+++ b/buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java
@@ -77,9 +77,9 @@
 import java.util.stream.Stream;
 
 public class DistroTestPlugin implements Plugin<Project> {
-    private static final String SYSTEM_JDK_VERSION = "21.0.2+13";
+    private static final String SYSTEM_JDK_VERSION = "21.0.3+9";
     private static final String SYSTEM_JDK_VENDOR = "adoptium";
-    private static final String GRADLE_JDK_VERSION = "21.0.2+13";
+    private static final String GRADLE_JDK_VERSION = "21.0.3+9";
     private static final String GRADLE_JDK_VENDOR = "adoptium";
 
     // all distributions used by distro tests. this is temporary until tests are per distribution
diff --git a/buildSrc/src/testFixtures/java/org/opensearch/gradle/test/GradleThreadsFilter.java b/buildSrc/src/testFixtures/java/org/opensearch/gradle/test/GradleThreadsFilter.java
index def5248c1f255..0ede465439400 100644
--- a/buildSrc/src/testFixtures/java/org/opensearch/gradle/test/GradleThreadsFilter.java
+++ b/buildSrc/src/testFixtures/java/org/opensearch/gradle/test/GradleThreadsFilter.java
@@ -45,6 +45,8 @@ public class GradleThreadsFilter implements ThreadFilter {
     public boolean reject(Thread t) {
         return t.getName().startsWith("Exec process")
             || t.getName().startsWith("Memory manager")
-            || t.getName().startsWith("File watcher consumer");
+            || t.getName().startsWith("File watcher consumer")
+            || t.getName().startsWith("sshd-SshClient") /* Started by SshClient (sshd-core), part of SftpFileSystemProvider */
+            || t.getName().startsWith("Thread-"); /* Started by AbstractFactoryManager (sshd-core), part of SftpFileSystemProvider */
     }
 }
diff --git a/buildSrc/version.properties b/buildSrc/version.properties
index 8705588babe97..d0aaea546803b 100644
--- a/buildSrc/version.properties
+++ b/buildSrc/version.properties
@@ -1,8 +1,8 @@
 opensearch        = 3.0.0
-lucene            = 9.11.0-snapshot-8a555eb
+lucene            = 9.11.0-snapshot-fb97840
 
 bundled_jdk_vendor = adoptium
-bundled_jdk = 21.0.2+13
+bundled_jdk = 21.0.3+9
 
 # optional dependencies
 spatial4j         = 0.7
@@ -14,7 +14,7 @@ icu4j             = 70.1
 supercsv          = 2.4.0
 log4j             = 2.21.0
 slf4j             = 1.7.36
-asm               = 9.6
+asm               = 9.7
 jettison          = 1.5.4
 woodstox          = 6.4.0
 kotlin            = 1.7.10
@@ -22,12 +22,13 @@ antlr4            = 4.13.1
 guava             = 32.1.1-jre
 protobuf          = 3.22.3
 jakarta_annotation = 1.3.5
+google_http_client = 1.44.1
 
 # when updating the JNA version, also update the version in buildSrc/build.gradle
 jna               = 5.13.0
 
-netty             = 4.1.107.Final
-joda              = 2.12.2
+netty             = 4.1.109.Final
+joda              = 2.12.7
 
 # project reactor
 reactor_netty     = 1.1.17
@@ -40,9 +41,10 @@ httpclient        = 4.5.14
 httpcore          = 4.4.16
 httpasyncclient   = 4.1.5
 commonslogging    = 1.2
-commonscodec      = 1.15
-commonslang       = 3.13.0
-commonscompress   = 1.24.0
+commonscodec      = 1.16.1
+commonslang       = 3.14.0
+commonscompress   = 1.26.1
+commonsio         = 2.16.0
 # plugin dependencies
 aws               = 2.20.86
 reactivestreams   = 1.0.4
@@ -50,7 +52,7 @@ reactivestreams   = 1.0.4
 # when updating this version, you need to ensure compatibility with:
 #  - plugins/ingest-attachment (transitive dependency, check the upstream POM)
 #  - distribution/tools/plugin-cli
-bouncycastle=1.77
+bouncycastle=1.78
 # test dependencies
 randomizedrunner  = 2.7.1
 junit             = 4.13.2
diff --git a/client/rest-high-level/src/test/java/org/opensearch/client/SearchIT.java b/client/rest-high-level/src/test/java/org/opensearch/client/SearchIT.java
index f2778a97c0c1a..c1f1cbf1d0e91 100644
--- a/client/rest-high-level/src/test/java/org/opensearch/client/SearchIT.java
+++ b/client/rest-high-level/src/test/java/org/opensearch/client/SearchIT.java
@@ -54,15 +54,19 @@
 import org.opensearch.action.search.SearchScrollRequest;
 import org.opensearch.client.core.CountRequest;
 import org.opensearch.client.core.CountResponse;
+import org.opensearch.common.geo.ShapeRelation;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.xcontent.XContentFactory;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.core.rest.RestStatus;
 import org.opensearch.core.xcontent.MediaTypeRegistry;
 import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.geometry.Rectangle;
+import org.opensearch.index.query.GeoShapeQueryBuilder;
 import org.opensearch.index.query.MatchQueryBuilder;
 import org.opensearch.index.query.QueryBuilder;
 import org.opensearch.index.query.QueryBuilders;
+import org.opensearch.index.query.RangeQueryBuilder;
 import org.opensearch.index.query.ScriptQueryBuilder;
 import org.opensearch.index.query.TermsQueryBuilder;
 import org.opensearch.join.aggregations.Children;
@@ -102,6 +106,8 @@
 import org.opensearch.search.suggest.Suggest;
 import org.opensearch.search.suggest.SuggestBuilder;
 import org.opensearch.search.suggest.phrase.PhraseSuggestionBuilder;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
 import org.hamcrest.Matchers;
 import org.junit.Before;
 
@@ -116,6 +122,7 @@
 import java.util.concurrent.TimeUnit;
 
 import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.opensearch.index.query.QueryBuilders.geoShapeQuery;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertToXContentEquivalent;
 import static org.hamcrest.Matchers.arrayContaining;
 import static org.hamcrest.Matchers.both;
@@ -764,6 +771,228 @@ public void testSearchWithWeirdScriptFields() throws Exception {
         }
     }
 
+    public void testSearchWithDerivedFields() throws Exception {
+        // Just testing DerivedField definition from SearchSourceBuilder derivedField()
+        // We are not testing the full functionality here
+        Request doc = new Request("PUT", "test/_doc/1");
+        doc.setJsonEntity("{\"field\":\"value\"}");
+        client().performRequest(doc);
+        client().performRequest(new Request("POST", "/test/_refresh"));
+        // Keyword field
+        {
+            SearchRequest searchRequest = new SearchRequest("test").source(
+                SearchSourceBuilder.searchSource()
+                    .derivedField("result", "keyword", new Script("emit(params._source[\"field\"])"))
+                    .fetchField("result")
+                    .query(new TermsQueryBuilder("result", "value"))
+            );
+            SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
+            SearchHit searchHit = searchResponse.getHits().getAt(0);
+            List<Object> values = searchHit.getFields().get("result").getValues();
+            assertNotNull(values);
+            assertEquals(1, values.size());
+            assertEquals("value", values.get(0));
+
+            // multi valued
+            searchRequest = new SearchRequest("test").source(
+                SearchSourceBuilder.searchSource()
+                    .derivedField(
+                        "result",
+                        "keyword",
+                        new Script("emit(params._source[\"field\"]);emit(params._source[\"field\"] + \"_2\")")
+                    )
+                    .query(new TermsQueryBuilder("result", "value_2"))
+                    .fetchField("result")
+            );
+            searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
+            searchHit = searchResponse.getHits().getAt(0);
+            values = searchHit.getFields().get("result").getValues();
+            assertNotNull(values);
+            assertEquals(2, values.size());
+            assertEquals("value", values.get(0));
+            assertEquals("value_2", values.get(1));
+        }
+        // Boolean field
+        {
+            SearchRequest searchRequest = new SearchRequest("test").source(
+                SearchSourceBuilder.searchSource()
+                    .derivedField("result", "boolean", new Script("emit(((String)params._source[\"field\"]).equals(\"value\"))"))
+                    .query(new TermsQueryBuilder("result", "true"))
+                    .fetchField("result")
+            );
+            SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
+            SearchHit searchHit = searchResponse.getHits().getAt(0);
+            List<Object> values = searchHit.getFields().get("result").getValues();
+            assertNotNull(values);
+            assertEquals(1, values.size());
+            assertEquals(true, values.get(0));
+        }
+        // Long field
+        {
+            SearchRequest searchRequest = new SearchRequest("test").source(
+                SearchSourceBuilder.searchSource()
+                    .derivedField("result", "long", new Script("emit(Long.MAX_VALUE)"))
+                    .query(new RangeQueryBuilder("result").from(Long.MAX_VALUE - 1).to(Long.MAX_VALUE))
+                    .fetchField("result")
+            );
+
+            SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
+            SearchHit searchHit = searchResponse.getHits().getAt(0);
+            List<Object> values = searchHit.getFields().get("result").getValues();
+            assertNotNull(values);
+            assertEquals(1, values.size());
+            assertEquals(Long.MAX_VALUE, values.get(0));
+
+            // multi-valued
+            searchRequest = new SearchRequest("test").source(
+                SearchSourceBuilder.searchSource()
+                    .derivedField("result", "long", new Script("emit(Long.MAX_VALUE); emit(Long.MIN_VALUE);"))
+                    .query(new RangeQueryBuilder("result").from(Long.MIN_VALUE).to(Long.MIN_VALUE + 1))
+                    .fetchField("result")
+            );
+
+            searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
+            searchHit = searchResponse.getHits().getAt(0);
+            values = searchHit.getFields().get("result").getValues();
+            assertNotNull(values);
+            assertEquals(2, values.size());
+            assertEquals(Long.MAX_VALUE, values.get(0));
+            assertEquals(Long.MIN_VALUE, values.get(1));
+        }
+        // Double field
+        {
+            SearchRequest searchRequest = new SearchRequest("test").source(
+                SearchSourceBuilder.searchSource()
+                    .derivedField("result", "double", new Script("emit(Double.MAX_VALUE)"))
+                    .query(new RangeQueryBuilder("result").from(Double.MAX_VALUE - 1).to(Double.MAX_VALUE))
+                    .fetchField("result")
+            );
+            SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
+            SearchHit searchHit = searchResponse.getHits().getAt(0);
+            List<Object> values = searchHit.getFields().get("result").getValues();
+            assertNotNull(values);
+            assertEquals(1, values.size());
+            assertEquals(Double.MAX_VALUE, values.get(0));
+
+            // multi-valued
+            searchRequest = new SearchRequest("test").source(
+                SearchSourceBuilder.searchSource()
+                    .derivedField("result", "double", new Script("emit(Double.MAX_VALUE); emit(Double.MIN_VALUE);"))
+                    .query(new RangeQueryBuilder("result").from(Double.MIN_VALUE).to(Double.MIN_VALUE + 1))
+                    .fetchField("result")
+            );
+
+            searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
+            searchHit = searchResponse.getHits().getAt(0);
+            values = searchHit.getFields().get("result").getValues();
+            assertNotNull(values);
+            assertEquals(2, values.size());
+            assertEquals(Double.MAX_VALUE, values.get(0));
+            assertEquals(Double.MIN_VALUE, values.get(1));
+        }
+        // Date field
+        {
+            DateTime date1 = new DateTime(1990, 12, 29, 0, 0, DateTimeZone.UTC);
+            DateTime date2 = new DateTime(1990, 12, 30, 0, 0, DateTimeZone.UTC);
+            SearchRequest searchRequest = new SearchRequest("test").source(
+                SearchSourceBuilder.searchSource()
+                    .derivedField("result", "date", new Script("emit(" + date1.getMillis() + "L)"))
+                    .query(new RangeQueryBuilder("result").from(date1.toString()).to(date2.toString()))
+                    .fetchField("result")
+            );
+
+            SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
+            SearchHit searchHit = searchResponse.getHits().getAt(0);
+            List<Object> values = searchHit.getFields().get("result").getValues();
+            assertNotNull(values);
+            assertEquals(1, values.size());
+            assertEquals(date1.toString(), values.get(0));
+
+            // multi-valued
+            searchRequest = new SearchRequest("test").source(
+                SearchSourceBuilder.searchSource()
+                    .derivedField("result", "date", new Script("emit(" + date1.getMillis() + "L); " + "emit(" + date2.getMillis() + "L)"))
+                    .query(new RangeQueryBuilder("result").from(date1.toString()).to(date2.toString()))
+                    .fetchField("result")
+            );
+
+            searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
+            searchHit = searchResponse.getHits().getAt(0);
+            values = searchHit.getFields().get("result").getValues();
+            assertNotNull(values);
+            assertEquals(2, values.size());
+            assertEquals(date1.toString(), values.get(0));
+            assertEquals(date2.toString(), values.get(1));
+        }
+        // Geo field
+        {
+            GeoShapeQueryBuilder qb = geoShapeQuery("result", new Rectangle(-35, 35, 35, -35));
+            qb.relation(ShapeRelation.INTERSECTS);
+            SearchRequest searchRequest = new SearchRequest("test").source(
+                SearchSourceBuilder.searchSource()
+                    .derivedField("result", "geo_point", new Script("emit(10.0, 20.0)"))
+                    .query(qb)
+                    .fetchField("result")
+            );
+
+            SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
+            SearchHit searchHit = searchResponse.getHits().getAt(0);
+            List<Object> values = searchHit.getFields().get("result").getValues();
+            assertNotNull(values);
+            assertEquals(1, values.size());
+            assertEquals(10.0, ((HashMap) values.get(0)).get("lat"));
+            assertEquals(20.0, ((HashMap) values.get(0)).get("lon"));
+
+            // multi-valued
+            searchRequest = new SearchRequest("test").source(
+                SearchSourceBuilder.searchSource()
+                    .derivedField("result", "geo_point", new Script("emit(10.0, 20.0); emit(20.0, 30.0);"))
+                    .query(qb)
+                    .fetchField("result")
+            );
+
+            searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
+            searchHit = searchResponse.getHits().getAt(0);
+            values = searchHit.getFields().get("result").getValues();
+            assertNotNull(values);
+            assertEquals(2, values.size());
+            assertEquals(10.0, ((HashMap) values.get(0)).get("lat"));
+            assertEquals(20.0, ((HashMap) values.get(0)).get("lon"));
+            assertEquals(20.0, ((HashMap) values.get(1)).get("lat"));
+            assertEquals(30.0, ((HashMap) values.get(1)).get("lon"));
+        }
+        // IP field
+        {
+            SearchRequest searchRequest = new SearchRequest("test").source(
+                SearchSourceBuilder.searchSource().derivedField("result", "ip", new Script("emit(\"10.0.0.1\")")).fetchField("result")
+            );
+
+            SearchResponse searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
+            SearchHit searchHit = searchResponse.getHits().getAt(0);
+            List<Object> values = searchHit.getFields().get("result").getValues();
+            assertNotNull(values);
+            assertEquals(1, values.size());
+            assertEquals("10.0.0.1", values.get(0));
+
+            // multi-valued
+            searchRequest = new SearchRequest("test").source(
+                SearchSourceBuilder.searchSource()
+                    .derivedField("result", "ip", new Script("emit(\"10.0.0.1\"); emit(\"10.0.0.2\");"))
+                    .fetchField("result")
+            );
+
+            searchResponse = execute(searchRequest, highLevelClient()::search, highLevelClient()::searchAsync);
+            searchHit = searchResponse.getHits().getAt(0);
+            values = searchHit.getFields().get("result").getValues();
+            assertNotNull(values);
+            assertEquals(2, values.size());
+            assertEquals("10.0.0.1", values.get(0));
+            assertEquals("10.0.0.2", values.get(1));
+
+        }
+
+    }
+
     public void testSearchScroll() throws Exception {
         for (int i = 0; i < 100; i++) {
             XContentBuilder builder = jsonBuilder().startObject().field("field", i).endObject();
diff --git a/client/rest/licenses/commons-codec-1.15.jar.sha1 b/client/rest/licenses/commons-codec-1.15.jar.sha1
deleted file mode 100644
index 62d99837b87e1..0000000000000
--- a/client/rest/licenses/commons-codec-1.15.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-49d94806b6e3dc933dacbd8acb0fdbab8ebd1e5d
\ No newline at end of file
diff --git a/client/rest/licenses/commons-codec-1.16.1.jar.sha1 b/client/rest/licenses/commons-codec-1.16.1.jar.sha1
new file mode 100644
index 0000000000000..6b8803089c6d7
--- /dev/null
+++ b/client/rest/licenses/commons-codec-1.16.1.jar.sha1
@@ -0,0 +1 @@
+47bd4d333fba53406f6c6c51884ddbca435c8862
\ No newline at end of file
diff --git a/client/rest/src/main/java/org/opensearch/client/nio/HttpEntityAsyncEntityProducer.java b/client/rest/src/main/java/org/opensearch/client/nio/HttpEntityAsyncEntityProducer.java
index 81fe77ddcfbed..4e6fd6f3d6f9d 100644
--- a/client/rest/src/main/java/org/opensearch/client/nio/HttpEntityAsyncEntityProducer.java
+++ b/client/rest/src/main/java/org/opensearch/client/nio/HttpEntityAsyncEntityProducer.java
@@ -16,6 +16,7 @@
 import org.apache.hc.core5.util.Asserts;
 
 import java.io.IOException;
+import java.nio.Buffer;
 import java.nio.ByteBuffer;
 import java.nio.channels.Channels;
 import java.nio.channels.ReadableByteChannel;
@@ -141,7 +142,7 @@ public void produce(final DataStreamChannel channel) throws IOException {
             }
         }
         if (byteBuffer.position() > 0) {
-            byteBuffer.flip();
+            ((Buffer) byteBuffer).flip();
             channel.write(byteBuffer);
             byteBuffer.compact();
         }
diff --git a/client/sniffer/licenses/commons-codec-1.15.jar.sha1 b/client/sniffer/licenses/commons-codec-1.15.jar.sha1
deleted file mode 100644
index 62d99837b87e1..0000000000000
--- a/client/sniffer/licenses/commons-codec-1.15.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-49d94806b6e3dc933dacbd8acb0fdbab8ebd1e5d
\ No newline at end of file
diff --git a/client/sniffer/licenses/commons-codec-1.16.1.jar.sha1 b/client/sniffer/licenses/commons-codec-1.16.1.jar.sha1
new file mode 100644
index 0000000000000..6b8803089c6d7
--- /dev/null
+++ b/client/sniffer/licenses/commons-codec-1.16.1.jar.sha1
@@ -0,0 +1 @@
+47bd4d333fba53406f6c6c51884ddbca435c8862
\ No newline at end of file
diff --git a/distribution/packages/build.gradle b/distribution/packages/build.gradle
index 4e85d19986e43..fbd13f03af814 100644
--- a/distribution/packages/build.gradle
+++ b/distribution/packages/build.gradle
@@ -63,7 +63,7 @@ import java.util.regex.Pattern
  */
 
 plugins {
-  id "com.netflix.nebula.ospackage-base" version "11.8.1"
+  id "com.netflix.nebula.ospackage-base" version "11.9.0"
 }
 
 void addProcessFilesTask(String type, boolean jdk) {
diff --git a/distribution/src/bin/opensearch b/distribution/src/bin/opensearch
index 947d1167f79f2..8a3b0a009437f 100755
--- a/distribution/src/bin/opensearch
+++ b/distribution/src/bin/opensearch
@@ -36,14 +36,16 @@ fi
 
 # get keystore password before setting java options to avoid
 # conflicting GC configurations for the keystore tools
-unset KEYSTORE_PASSWORD
-KEYSTORE_PASSWORD=
 if [[ $CHECK_KEYSTORE = true ]] \
     && bin/opensearch-keystore has-passwd --silent
 then
-  if ! read -s -r -p "OpenSearch keystore password: " KEYSTORE_PASSWORD ; then
-    echo "Failed to read keystore password on console" 1>&2
-    exit 1
+  if [[ ! -z "${KEYSTORE_PASSWORD}" ]]; then
+    echo "Using value of KEYSTORE_PASSWORD from the environment"
+  else
+    if ! read -s -r -p "OpenSearch keystore password: " KEYSTORE_PASSWORD ; then
+      echo "Failed to read keystore password on console" 1>&2
+      exit 1
+    fi
   fi
 fi
 
diff --git a/distribution/src/bin/opensearch.bat b/distribution/src/bin/opensearch.bat
index cce21504c55b7..b7ecab24165fa 100644
--- a/distribution/src/bin/opensearch.bat
+++ b/distribution/src/bin/opensearch.bat
@@ -62,14 +62,17 @@ if not exist "%SERVICE_LOG_DIR%" (
 	mkdir "%SERVICE_LOG_DIR%"
 )
 
-SET KEYSTORE_PASSWORD=
 IF "%checkpassword%"=="Y" (
   CALL "%~dp0opensearch-keystore.bat" has-passwd --silent
   IF !ERRORLEVEL! EQU 0 (
-    SET /P KEYSTORE_PASSWORD=OpenSearch keystore password:
-    IF !ERRORLEVEL! NEQ 0 (
-      ECHO Failed to read keystore password on standard input
-      EXIT /B !ERRORLEVEL!
+    if defined KEYSTORE_PASSWORD (
+      ECHO Using value of KEYSTORE_PASSWORD from the environment
+    ) else (
+      SET /P KEYSTORE_PASSWORD=OpenSearch keystore password:
+      IF !ERRORLEVEL! NEQ 0 (
+        ECHO Failed to read keystore password on standard input
+        EXIT /B !ERRORLEVEL!
+      )
     )
   )
 )
diff --git a/distribution/tools/plugin-cli/build.gradle b/distribution/tools/plugin-cli/build.gradle
index f40fb1c4b0a9f..3083ad4375460 100644
--- a/distribution/tools/plugin-cli/build.gradle
+++ b/distribution/tools/plugin-cli/build.gradle
@@ -38,13 +38,13 @@ dependencies {
   compileOnly project(":server")
   compileOnly project(":libs:opensearch-cli")
   api "org.bouncycastle:bcpg-fips:1.0.7.1"
-  api "org.bouncycastle:bc-fips:1.0.2.4"
+  api "org.bouncycastle:bc-fips:1.0.2.5"
   testImplementation project(":test:framework")
   testImplementation 'com.google.jimfs:jimfs:1.3.0'
   testRuntimeOnly("com.google.guava:guava:${versions.guava}") {
     transitive = false
   }
-
+  api "commons-io:commons-io:${versions.commonsio}"
   implementation "org.apache.commons:commons-compress:${versions.commonscompress}"
 }
 
@@ -104,5 +104,8 @@ thirdPartyAudit.ignoreMissingClasses(
   'org.tukaani.xz.MemoryLimitException',
   'org.tukaani.xz.UnsupportedOptionsException',
   'org.tukaani.xz.XZ',
-  'org.tukaani.xz.XZOutputStream'
+  'org.tukaani.xz.XZOutputStream',
+  'org.apache.commons.codec.digest.PureJavaCrc32C',
+  'org.apache.commons.codec.digest.XXHash32',
+  'org.apache.commons.lang3.reflect.FieldUtils'
 )
diff --git a/distribution/tools/plugin-cli/licenses/bc-fips-1.0.2.4.jar.sha1 b/distribution/tools/plugin-cli/licenses/bc-fips-1.0.2.4.jar.sha1
deleted file mode 100644
index da37449f80d7e..0000000000000
--- a/distribution/tools/plugin-cli/licenses/bc-fips-1.0.2.4.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-9008d04fc13da6455e6a792935b93b629757335d
\ No newline at end of file
diff --git a/distribution/tools/plugin-cli/licenses/bc-fips-1.0.2.5.jar.sha1 b/distribution/tools/plugin-cli/licenses/bc-fips-1.0.2.5.jar.sha1
new file mode 100644
index 0000000000000..1b44c77dd4ee1
--- /dev/null
+++ b/distribution/tools/plugin-cli/licenses/bc-fips-1.0.2.5.jar.sha1
@@ -0,0 +1 @@
+704e65f7e4fe679e5ab2aa8a840f27f8ced4c522
\ No newline at end of file
diff --git a/distribution/tools/plugin-cli/licenses/commons-compress-1.24.0.jar.sha1 b/distribution/tools/plugin-cli/licenses/commons-compress-1.24.0.jar.sha1
deleted file mode 100644
index 23999d1bfbde4..0000000000000
--- a/distribution/tools/plugin-cli/licenses/commons-compress-1.24.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b4b1b5a3d9573b2970fddab236102c0a4d27d35e
\ No newline at end of file
diff --git a/distribution/tools/plugin-cli/licenses/commons-compress-1.26.1.jar.sha1 b/distribution/tools/plugin-cli/licenses/commons-compress-1.26.1.jar.sha1
new file mode 100644
index 0000000000000..912bda85de18a
--- /dev/null
+++ b/distribution/tools/plugin-cli/licenses/commons-compress-1.26.1.jar.sha1
@@ -0,0 +1 @@
+44331c1130c370e726a2e1a3e6fba6d2558ef04a
\ No newline at end of file
diff --git a/distribution/tools/plugin-cli/licenses/commons-io-2.16.0.jar.sha1 b/distribution/tools/plugin-cli/licenses/commons-io-2.16.0.jar.sha1
new file mode 100644
index 0000000000000..6a7b638719fa3
--- /dev/null
+++ b/distribution/tools/plugin-cli/licenses/commons-io-2.16.0.jar.sha1
@@ -0,0 +1 @@
+27875a7935f1ddcc13267eb6fae1f719e0409572
\ No newline at end of file
diff --git a/distribution/tools/plugin-cli/licenses/commons-io-LICENSE.txt b/distribution/tools/plugin-cli/licenses/commons-io-LICENSE.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/distribution/tools/plugin-cli/licenses/commons-io-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/distribution/tools/plugin-cli/licenses/commons-io-NOTICE.txt b/distribution/tools/plugin-cli/licenses/commons-io-NOTICE.txt
new file mode 100644
index 0000000000000..a6b77d1eb6089
--- /dev/null
+++ b/distribution/tools/plugin-cli/licenses/commons-io-NOTICE.txt
@@ -0,0 +1,5 @@
+Apache Commons IO
+Copyright 2002-2014 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
diff --git a/doc-tools/missing-doclet/src/main/java/org/opensearch/missingdoclet/MissingDoclet.java b/doc-tools/missing-doclet/src/main/java/org/opensearch/missingdoclet/MissingDoclet.java
index e1ad55fe4b60b..470ab75b7930c 100644
--- a/doc-tools/missing-doclet/src/main/java/org/opensearch/missingdoclet/MissingDoclet.java
+++ b/doc-tools/missing-doclet/src/main/java/org/opensearch/missingdoclet/MissingDoclet.java
@@ -332,13 +332,20 @@ private void checkComment(Element element) {
 
     // Ignore classes annotated with @Generated and all enclosed elements in them.
     private boolean isGenerated(Element element) {
-        return element
+        final boolean isGenerated = element
             .getAnnotationMirrors()
             .stream()
             .anyMatch(m -> m
                 .getAnnotationType()
                 .toString() /* ClassSymbol.toString() returns class name */
                 .equalsIgnoreCase("javax.annotation.Generated"));
+
+        if (!isGenerated && element.getEnclosingElement() != null) {
+            // check if enclosing element is generated
+            return isGenerated(element.getEnclosingElement());
+        } 
+
+        return isGenerated;
     }
 
     private boolean hasInheritedJavadocs(Element element) {
diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
index 82a4add334a7d..9b0d73222260e 100644
--- a/gradle/wrapper/gradle-wrapper.properties
+++ b/gradle/wrapper/gradle-wrapper.properties
@@ -11,7 +11,7 @@
 
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.6-all.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-all.zip
 zipStoreBase=GRADLE_USER_HOME
 zipStorePath=wrapper/dists
-distributionSha256Sum=85719317abd2112f021d4f41f09ec370534ba288432065f4b477b6a3b652910d
+distributionSha256Sum=194717442575a6f96e1c1befa2c30e9a4fc90f701d7aee33eb879b79e7ff05c0
diff --git a/libs/common/src/main/java/org/opensearch/common/Glob.java b/libs/common/src/main/java/org/opensearch/common/Glob.java
index daf045dd49e3a..b390a3ca84182 100644
--- a/libs/common/src/main/java/org/opensearch/common/Glob.java
+++ b/libs/common/src/main/java/org/opensearch/common/Glob.java
@@ -52,34 +52,35 @@ public static boolean globMatch(String pattern, String str) {
         if (pattern == null || str == null) {
             return false;
         }
-        int firstIndex = pattern.indexOf('*');
-        if (firstIndex == -1) {
-            return pattern.equals(str);
-        }
-        if (firstIndex == 0) {
-            if (pattern.length() == 1) {
-                return true;
-            }
-            int nextIndex = pattern.indexOf('*', firstIndex + 1);
-            if (nextIndex == -1) {
-                return str.endsWith(pattern.substring(1));
-            } else if (nextIndex == 1) {
-                // Double wildcard "**" - skipping the first "*"
-                return globMatch(pattern.substring(1), str);
+        int sIdx = 0, pIdx = 0, match = 0, wildcardIdx = -1;
+        while (sIdx < str.length()) {
+            // both chars matching, incrementing both pointers
+            if (pIdx < pattern.length() && str.charAt(sIdx) == pattern.charAt(pIdx)) {
+                sIdx++;
+                pIdx++;
+            } else if (pIdx < pattern.length() && pattern.charAt(pIdx) == '*') {
+                // wildcard found, only incrementing pattern pointer
+                wildcardIdx = pIdx;
+                match = sIdx;
+                pIdx++;
+            } else if (wildcardIdx != -1) {
+                // last pattern pointer was a wildcard, incrementing string pointer
+                pIdx = wildcardIdx + 1;
+                match++;
+                sIdx = match;
+            } else {
+                // current pattern pointer is not a wildcard, last pattern pointer was also not a wildcard
+                // characters do not match
+                return false;
             }
-            String part = pattern.substring(1, nextIndex);
-            int partIndex = str.indexOf(part);
-            while (partIndex != -1) {
-                if (globMatch(pattern.substring(nextIndex), str.substring(partIndex + part.length()))) {
-                    return true;
-                }
-                partIndex = str.indexOf(part, partIndex + 1);
-            }
-            return false;
         }
-        return (str.length() >= firstIndex
-            && pattern.substring(0, firstIndex).equals(str.substring(0, firstIndex))
-            && globMatch(pattern.substring(firstIndex), str.substring(firstIndex)));
+
+        // check for remaining characters in pattern
+        while (pIdx < pattern.length() && pattern.charAt(pIdx) == '*') {
+            pIdx++;
+        }
+
+        return pIdx == pattern.length();
     }
 
 }
diff --git a/libs/core/licenses/lucene-core-9.11.0-snapshot-8a555eb.jar.sha1 b/libs/core/licenses/lucene-core-9.11.0-snapshot-8a555eb.jar.sha1
deleted file mode 100644
index 25beb34873c0c..0000000000000
--- a/libs/core/licenses/lucene-core-9.11.0-snapshot-8a555eb.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-fe1cf5663be8bdb6aa757fd4101bc551684c90fb
\ No newline at end of file
diff --git a/libs/core/licenses/lucene-core-9.11.0-snapshot-fb97840.jar.sha1 b/libs/core/licenses/lucene-core-9.11.0-snapshot-fb97840.jar.sha1
new file mode 100644
index 0000000000000..88309bc46411a
--- /dev/null
+++ b/libs/core/licenses/lucene-core-9.11.0-snapshot-fb97840.jar.sha1
@@ -0,0 +1 @@
+53a828e3e88f55c83979cd3df0704617cc9edb9a
\ No newline at end of file
diff --git a/libs/core/src/main/java/org/opensearch/OpenSearchException.java b/libs/core/src/main/java/org/opensearch/OpenSearchException.java
index cce86b452f698..dda3983fbb4d1 100644
--- a/libs/core/src/main/java/org/opensearch/OpenSearchException.java
+++ b/libs/core/src/main/java/org/opensearch/OpenSearchException.java
@@ -33,6 +33,7 @@
 
 import org.opensearch.common.CheckedFunction;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.collect.Tuple;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.common.Strings;
@@ -69,8 +70,9 @@
 /**
  * A core library base class for all opensearch exceptions.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class OpenSearchException extends RuntimeException implements Writeable, ToXContentFragment {
 
     protected static final Version UNKNOWN_VERSION_ADDED = Version.fromId(0);
diff --git a/libs/core/src/main/java/org/opensearch/Version.java b/libs/core/src/main/java/org/opensearch/Version.java
index 56df46ae94d44..823eb110eaa10 100644
--- a/libs/core/src/main/java/org/opensearch/Version.java
+++ b/libs/core/src/main/java/org/opensearch/Version.java
@@ -101,7 +101,9 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_2_12_0 = new Version(2120099, org.apache.lucene.util.Version.LUCENE_9_9_2);
     public static final Version V_2_12_1 = new Version(2120199, org.apache.lucene.util.Version.LUCENE_9_9_2);
     public static final Version V_2_13_0 = new Version(2130099, org.apache.lucene.util.Version.LUCENE_9_10_0);
+    public static final Version V_2_13_1 = new Version(2130199, org.apache.lucene.util.Version.LUCENE_9_10_0);
     public static final Version V_2_14_0 = new Version(2140099, org.apache.lucene.util.Version.LUCENE_9_10_0);
+    public static final Version V_2_15_0 = new Version(2150099, org.apache.lucene.util.Version.LUCENE_9_10_0);
     public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_11_0);
     public static final Version CURRENT = V_3_0_0;
 
diff --git a/libs/core/src/main/java/org/opensearch/core/common/breaker/CircuitBreaker.java b/libs/core/src/main/java/org/opensearch/core/common/breaker/CircuitBreaker.java
index 846950ff17c63..9a09b3b38a5f2 100644
--- a/libs/core/src/main/java/org/opensearch/core/common/breaker/CircuitBreaker.java
+++ b/libs/core/src/main/java/org/opensearch/core/common/breaker/CircuitBreaker.java
@@ -32,14 +32,17 @@
 
 package org.opensearch.core.common.breaker;
 
+import org.opensearch.common.annotation.PublicApi;
+
 import java.util.Locale;
 
 /**
  * Interface for an object that can be incremented, breaking after some
  * configured limit has been reached.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface CircuitBreaker {
 
     /**
@@ -72,8 +75,10 @@ public interface CircuitBreaker {
     /**
      * The type of breaker
      * can be {@link #MEMORY}, {@link #PARENT}, or {@link #NOOP}
-     * @opensearch.internal
+     *
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     enum Type {
         /** A regular or ChildMemoryCircuitBreaker */
         MEMORY,
diff --git a/libs/core/src/main/java/org/opensearch/core/common/io/stream/FilterStreamInput.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/FilterStreamInput.java
index a6e49567ac7d5..ee67fd4f271a2 100644
--- a/libs/core/src/main/java/org/opensearch/core/common/io/stream/FilterStreamInput.java
+++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/FilterStreamInput.java
@@ -80,6 +80,16 @@ public void reset() throws IOException {
         delegate.reset();
     }
 
+    @Override
+    public void mark(int readlimit) {
+        delegate.mark(readlimit);
+    }
+
+    @Override
+    public boolean markSupported() {
+        return delegate.markSupported();
+    }
+
     @Override
     public int read() throws IOException {
         return delegate.read();
diff --git a/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamInput.java b/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamInput.java
index ea23b3d81a775..f4c52cb8a6506 100644
--- a/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamInput.java
+++ b/libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamInput.java
@@ -80,6 +80,7 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.Date;
 import java.util.EnumSet;
 import java.util.HashMap;
@@ -90,6 +91,8 @@
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
+import java.util.SortedMap;
+import java.util.TreeMap;
 import java.util.concurrent.TimeUnit;
 import java.util.function.IntFunction;
 
@@ -642,12 +645,47 @@ public <K, V> Map<K, V> readMap(Writeable.Reader<K> keyReader, Writeable.Reader<
             return Collections.emptyMap();
         }
         Map<K, V> map = new HashMap<>(size);
+        readIntoMap(keyReader, valueReader, map, size);
+        return map;
+    }
+
+    /**
+     * Read a serialized map into a SortedMap using the default ordering for the keys. If the result is empty it might be immutable.
+     */
+    public <K extends Comparable<K>, V> SortedMap<K, V> readOrderedMap(Writeable.Reader<K> keyReader, Writeable.Reader<V> valueReader)
+        throws IOException {
+        return readOrderedMap(keyReader, valueReader, null);
+    }
+
+    /**
+     * Read a serialized map into a SortedMap, specifying a Comparator for the keys. If the result is empty it might be immutable.
+     */
+    public <K extends Comparable<K>, V> SortedMap<K, V> readOrderedMap(
+        Writeable.Reader<K> keyReader,
+        Writeable.Reader<V> valueReader,
+        @Nullable Comparator<K> keyComparator
+    ) throws IOException {
+        int size = readArraySize();
+        if (size == 0) {
+            return Collections.emptySortedMap();
+        }
+        SortedMap<K, V> sortedMap;
+        if (keyComparator == null) {
+            sortedMap = new TreeMap<>();
+        } else {
+            sortedMap = new TreeMap<>(keyComparator);
+        }
+        readIntoMap(keyReader, valueReader, sortedMap, size);
+        return sortedMap;
+    }
+
+    private <K, V> void readIntoMap(Writeable.Reader<K> keyReader, Writeable.Reader<V> valueReader, Map<K, V> map, int size)
+        throws IOException {
         for (int i = 0; i < size; i++) {
             K key = keyReader.read(this);
             V value = valueReader.read(this);
             map.put(key, value);
         }
-        return map;
     }
 
     /**
diff --git a/libs/core/src/main/java/org/opensearch/core/common/transport/BoundTransportAddress.java b/libs/core/src/main/java/org/opensearch/core/common/transport/BoundTransportAddress.java
index 8908a172395f2..e2266339c058f 100644
--- a/libs/core/src/main/java/org/opensearch/core/common/transport/BoundTransportAddress.java
+++ b/libs/core/src/main/java/org/opensearch/core/common/transport/BoundTransportAddress.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.core.common.transport;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.network.InetAddresses;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -44,8 +45,9 @@
  * the addresses the transport is bound to, and the other is the published one that represents the address clients
  * should communicate on.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class BoundTransportAddress implements Writeable {
 
     private TransportAddress[] boundAddresses;
diff --git a/libs/core/src/main/java/org/opensearch/core/transport/TransportResponse.java b/libs/core/src/main/java/org/opensearch/core/transport/TransportResponse.java
index 038069e93a51b..4ae01e140a89c 100644
--- a/libs/core/src/main/java/org/opensearch/core/transport/TransportResponse.java
+++ b/libs/core/src/main/java/org/opensearch/core/transport/TransportResponse.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.core.transport;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 
@@ -40,8 +41,9 @@
 /**
  * Response over the transport interface
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public abstract class TransportResponse extends TransportMessage {
 
     /**
diff --git a/libs/core/src/main/java/org/opensearch/semver/SemverRange.java b/libs/core/src/main/java/org/opensearch/semver/SemverRange.java
index da87acc7124aa..da8c06c07d8e5 100644
--- a/libs/core/src/main/java/org/opensearch/semver/SemverRange.java
+++ b/libs/core/src/main/java/org/opensearch/semver/SemverRange.java
@@ -10,6 +10,7 @@
 
 import org.opensearch.Version;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.xcontent.ToXContentFragment;
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.semver.expr.Caret;
@@ -31,7 +32,10 @@
  *     <li>'~' Allows for patch version variability starting from the range version. For example, ~1.2.3 range would match versions greater than or equal to 1.2.3 but less than 1.3.0</li>
  *     <li>'^' Allows for patch and minor version variability starting from the range version. For example, ^1.2.3 range would match versions greater than or equal to 1.2.3 but less than 2.0.0</li>
  * </ul>
+ *
+ * @opensearch.api
  */
+@PublicApi(since = "2.13.0")
 public class SemverRange implements ToXContentFragment {
 
     private final Version rangeVersion;
diff --git a/libs/core/src/test/java/org/opensearch/core/common/io/stream/FilterStreamInputTests.java b/libs/core/src/test/java/org/opensearch/core/common/io/stream/FilterStreamInputTests.java
index a044586e095e3..ab6dfbc2feb25 100644
--- a/libs/core/src/test/java/org/opensearch/core/common/io/stream/FilterStreamInputTests.java
+++ b/libs/core/src/test/java/org/opensearch/core/common/io/stream/FilterStreamInputTests.java
@@ -12,6 +12,9 @@
 import org.opensearch.core.common.bytes.BytesReference;
 
 import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import static org.hamcrest.Matchers.is;
 
 /** test the FilterStreamInput using the same BaseStreamTests */
 public class FilterStreamInputTests extends BaseStreamTests {
@@ -21,4 +24,24 @@ protected StreamInput getStreamInput(BytesReference bytesReference) throws IOExc
         return new FilterStreamInput(StreamInput.wrap(br.bytes, br.offset, br.length)) {
         };
     }
+
+    public void testMarkAndReset() throws IOException {
+        FilterStreamInputTests filterStreamInputTests = new FilterStreamInputTests();
+
+        ByteBuffer buffer = ByteBuffer.wrap(new byte[20]);
+        for (int i = 0; i < buffer.limit(); i++) {
+            buffer.put((byte) i);
+        }
+        buffer.rewind();
+        BytesReference bytesReference = BytesReference.fromByteBuffer(buffer);
+        StreamInput streamInput = filterStreamInputTests.getStreamInput(bytesReference);
+        streamInput.read();
+        assertThat(streamInput.markSupported(), is(true));
+        streamInput.mark(-1);
+        int int1 = streamInput.read();
+        int int2 = streamInput.read();
+        streamInput.reset();
+        assertEquals(int1, streamInput.read());
+        assertEquals(int2, streamInput.read());
+    }
 }
diff --git a/libs/x-content/src/test/java/org/opensearch/common/xcontent/XContentParserTests.java b/libs/x-content/src/test/java/org/opensearch/common/xcontent/XContentParserTests.java
index 81a2b0e290121..16816a03d742d 100644
--- a/libs/x-content/src/test/java/org/opensearch/common/xcontent/XContentParserTests.java
+++ b/libs/x-content/src/test/java/org/opensearch/common/xcontent/XContentParserTests.java
@@ -85,8 +85,8 @@ public class XContentParserTests extends OpenSearchTestCase {
         () -> randomAlphaOfLengthBetween(1, SmileXContent.DEFAULT_MAX_STRING_LEN),
         /* YAML parser limitation */
         XContentType.YAML,
-        /* use 75% of the limit, difficult to get the exact size of the content right */
-        () -> randomRealisticUnicodeOfCodepointLengthBetween(1, (int) (YamlXContent.DEFAULT_CODEPOINT_LIMIT * 0.75))
+        /* use 50% of the limit, difficult to get the exact size of the content right */
+        () -> randomRealisticUnicodeOfCodepointLengthBetween(1, (int) (YamlXContent.DEFAULT_CODEPOINT_LIMIT * 0.50))
     );
 
     private static final Map<XContentType, Supplier<String>> OFF_LIMIT_GENERATORS = Map.of(
diff --git a/modules/cache-common/src/internalClusterTest/java/org.opensearch.cache.common.tier/TieredSpilloverCacheIT.java b/modules/cache-common/src/internalClusterTest/java/org.opensearch.cache.common.tier/TieredSpilloverCacheIT.java
index 568ac4d188c51..bfc184cff0566 100644
--- a/modules/cache-common/src/internalClusterTest/java/org.opensearch.cache.common.tier/TieredSpilloverCacheIT.java
+++ b/modules/cache-common/src/internalClusterTest/java/org.opensearch.cache.common.tier/TieredSpilloverCacheIT.java
@@ -12,21 +12,31 @@
 import org.opensearch.action.admin.cluster.node.info.NodesInfoRequest;
 import org.opensearch.action.admin.cluster.node.info.NodesInfoResponse;
 import org.opensearch.action.admin.cluster.node.info.PluginsAndModules;
+import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
+import org.opensearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest;
+import org.opensearch.action.admin.indices.cache.clear.ClearIndicesCacheResponse;
+import org.opensearch.action.admin.indices.forcemerge.ForceMergeResponse;
 import org.opensearch.action.search.SearchResponse;
 import org.opensearch.action.search.SearchType;
 import org.opensearch.client.Client;
+import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.common.cache.CacheType;
 import org.opensearch.common.cache.ICache;
 import org.opensearch.common.cache.settings.CacheSettings;
 import org.opensearch.common.cache.store.OpenSearchOnHeapCache;
+import org.opensearch.common.cache.store.settings.OpenSearchOnHeapCacheSettings;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.util.FeatureFlags;
+import org.opensearch.index.cache.request.RequestCacheStats;
+import org.opensearch.index.query.QueryBuilders;
 import org.opensearch.indices.IndicesRequestCache;
 import org.opensearch.plugins.CachePlugin;
 import org.opensearch.plugins.Plugin;
 import org.opensearch.plugins.PluginInfo;
 import org.opensearch.search.aggregations.bucket.histogram.DateHistogramInterval;
 import org.opensearch.test.OpenSearchIntegTestCase;
+import org.opensearch.test.hamcrest.OpenSearchAssertions;
 import org.junit.Assert;
 
 import java.time.ZoneId;
@@ -34,15 +44,20 @@
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+import static org.opensearch.common.cache.store.settings.OpenSearchOnHeapCacheSettings.MAXIMUM_SIZE_IN_BYTES_KEY;
+import static org.opensearch.indices.IndicesService.INDICES_CACHE_CLEAN_INTERVAL_SETTING;
 import static org.opensearch.search.aggregations.AggregationBuilders.dateHistogram;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertNoFailures;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse;
 import static org.hamcrest.Matchers.greaterThan;
 
+@OpenSearchIntegTestCase.ClusterScope(numDataNodes = 0, scope = OpenSearchIntegTestCase.Scope.TEST)
 public class TieredSpilloverCacheIT extends OpenSearchIntegTestCase {
 
     @Override
@@ -50,15 +65,9 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
         return Arrays.asList(TieredSpilloverCachePlugin.class, MockDiskCachePlugin.class);
     }
 
-    @Override
-    protected Settings featureFlagSettings() {
-        return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.PLUGGABLE_CACHE, "true").build();
-    }
-
-    @Override
-    protected Settings nodeSettings(int nodeOrdinal) {
+    private Settings defaultSettings(String onHeapCacheSizeInBytesOrPecentage) {
         return Settings.builder()
-            .put(super.nodeSettings(nodeOrdinal))
+            .put(FeatureFlags.PLUGGABLE_CACHE, "true")
             .put(
                 CacheSettings.getConcreteStoreNameSettingForCacheType(CacheType.INDICES_REQUEST_CACHE).getKey(),
                 TieredSpilloverCache.TieredSpilloverCacheFactory.TIERED_SPILLOVER_CACHE_NAME
@@ -75,10 +84,17 @@ protected Settings nodeSettings(int nodeOrdinal) {
                 ).getKey(),
                 MockDiskCache.MockDiskCacheFactory.NAME
             )
+            .put(
+                OpenSearchOnHeapCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE)
+                    .get(MAXIMUM_SIZE_IN_BYTES_KEY)
+                    .getKey(),
+                onHeapCacheSizeInBytesOrPecentage
+            )
             .build();
     }
 
     public void testPluginsAreInstalled() {
+        internalCluster().startNode(Settings.builder().put(defaultSettings("1%")).build());
         NodesInfoRequest nodesInfoRequest = new NodesInfoRequest();
         nodesInfoRequest.addMetric(NodesInfoRequest.Metric.PLUGINS.metricName());
         NodesInfoResponse nodesInfoResponse = OpenSearchIntegTestCase.client().admin().cluster().nodesInfo(nodesInfoRequest).actionGet();
@@ -90,11 +106,12 @@ public void testPluginsAreInstalled() {
             .collect(Collectors.toList());
         Assert.assertTrue(
             pluginInfos.stream()
-                .anyMatch(pluginInfo -> pluginInfo.getName().equals("org.opensearch.cache.common" + ".tier.TieredSpilloverCachePlugin"))
+                .anyMatch(pluginInfo -> pluginInfo.getName().equals("org.opensearch.cache.common.tier.TieredSpilloverCachePlugin"))
         );
     }
 
     public void testSanityChecksWithIndicesRequestCache() throws InterruptedException {
+        internalCluster().startNodes(3, Settings.builder().put(defaultSettings("1%")).build());
         Client client = client();
         assertAcked(
             client.admin()
@@ -118,10 +135,7 @@ public void testSanityChecksWithIndicesRequestCache() throws InterruptedExceptio
             .setSize(0)
             .setSearchType(SearchType.QUERY_THEN_FETCH)
             .addAggregation(
-                dateHistogram("histo").field("f")
-                    .timeZone(ZoneId.of("+01:00"))
-                    .minDocCount(0)
-                    .dateHistogramInterval(DateHistogramInterval.MONTH)
+                dateHistogram("histo").field("f").timeZone(ZoneId.of("+01:00")).minDocCount(0).calendarInterval(DateHistogramInterval.MONTH)
             )
             .get();
         assertSearchResponse(r1);
@@ -133,13 +147,410 @@ public void testSanityChecksWithIndicesRequestCache() throws InterruptedExceptio
         );
     }
 
+    public void testWithDynamicTookTimePolicy() throws Exception {
+        int onHeapCacheSizeInBytes = 2000;
+        internalCluster().startNode(Settings.builder().put(defaultSettings(onHeapCacheSizeInBytes + "b")).build());
+        Client client = client();
+        assertAcked(
+            client.admin()
+                .indices()
+                .prepareCreate("index")
+                .setMapping("k", "type=keyword")
+                .setSettings(
+                    Settings.builder()
+                        .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
+                        .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                        .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                        .put("index.refresh_interval", -1)
+                )
+                .get()
+        );
+        // Step 1 : Set a very high value for took time policy so that no items evicted from onHeap cache are spilled
+        // to disk. And then hit requests so that few items are cached into cache.
+        ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest().transientSettings(
+            Settings.builder()
+                .put(
+                    TieredSpilloverCacheSettings.TOOK_TIME_POLICY_CONCRETE_SETTINGS_MAP.get(CacheType.INDICES_REQUEST_CACHE).getKey(),
+                    new TimeValue(100, TimeUnit.SECONDS)
+                )
+                .build()
+        );
+        assertAcked(internalCluster().client().admin().cluster().updateSettings(updateSettingsRequest).get());
+        int numberOfIndexedItems = randomIntBetween(6, 10);
+        for (int iterator = 0; iterator < numberOfIndexedItems; iterator++) {
+            indexRandom(true, client.prepareIndex("index").setSource("k" + iterator, "hello" + iterator));
+        }
+        ensureSearchable("index");
+        refreshAndWaitForReplication();
+        // Force merge the index to ensure there can be no background merges during the subsequent searches that would invalidate the cache
+        ForceMergeResponse forceMergeResponse = client.admin().indices().prepareForceMerge("index").setFlush(true).get();
+        OpenSearchAssertions.assertAllSuccessful(forceMergeResponse);
+        long perQuerySizeInCacheInBytes = -1;
+        for (int iterator = 0; iterator < numberOfIndexedItems; iterator++) {
+            SearchResponse resp = client.prepareSearch("index")
+                .setRequestCache(true)
+                .setQuery(QueryBuilders.termQuery("k" + iterator, "hello" + iterator))
+                .get();
+            if (perQuerySizeInCacheInBytes == -1) {
+                RequestCacheStats requestCacheStats = getRequestCacheStats(client, "index");
+                perQuerySizeInCacheInBytes = requestCacheStats.getMemorySizeInBytes();
+            }
+            assertSearchResponse(resp);
+        }
+        RequestCacheStats requestCacheStats = getRequestCacheStats(client, "index");
+        // Considering disk cache won't be used due to took time policy having a high value, we expect overall cache
+        // size to be less than or equal to onHeapCache size.
+        assertTrue(requestCacheStats.getMemorySizeInBytes() <= onHeapCacheSizeInBytes);
+        long entriesInCache = requestCacheStats.getMemorySizeInBytes() / perQuerySizeInCacheInBytes;
+        // All should be misses in the first attempt
+        assertEquals(numberOfIndexedItems, requestCacheStats.getMissCount());
+        assertEquals(numberOfIndexedItems - entriesInCache, requestCacheStats.getEvictions());
+        assertEquals(0, requestCacheStats.getHitCount());
+
+        // Step 2: Again hit same set of queries as above, we still won't see any hits as items keeps getting evicted.
+        for (int iterator = 0; iterator < numberOfIndexedItems; iterator++) {
+            SearchResponse resp = client.prepareSearch("index")
+                .setRequestCache(true)
+                .setQuery(QueryBuilders.termQuery("k" + iterator, "hello" + iterator))
+                .get();
+            assertSearchResponse(resp);
+        }
+        requestCacheStats = getRequestCacheStats(client, "index");
+        // We still won't get any hits as items keep getting evicted in LRU fashion due to limited cache size.
+        assertTrue(requestCacheStats.getMemorySizeInBytes() <= onHeapCacheSizeInBytes);
+        assertEquals(numberOfIndexedItems * 2, requestCacheStats.getMissCount());
+        assertEquals(numberOfIndexedItems * 2 - entriesInCache, requestCacheStats.getEvictions());
+        assertEquals(0, requestCacheStats.getHitCount());
+        long lastEvictionSeen = requestCacheStats.getEvictions();
+
+        // Step 3: Decrease took time policy to zero so that disk cache also comes into play. Now we should be able
+        // to cache all entries.
+        updateSettingsRequest = new ClusterUpdateSettingsRequest().transientSettings(
+            Settings.builder()
+                .put(
+                    TieredSpilloverCacheSettings.TOOK_TIME_POLICY_CONCRETE_SETTINGS_MAP.get(CacheType.INDICES_REQUEST_CACHE).getKey(),
+                    new TimeValue(0, TimeUnit.MILLISECONDS)
+                )
+                .build()
+        );
+        assertAcked(internalCluster().client().admin().cluster().updateSettings(updateSettingsRequest).get());
+        for (int iterator = 0; iterator < numberOfIndexedItems; iterator++) {
+            SearchResponse resp = client.prepareSearch("index")
+                .setRequestCache(true)
+                .setQuery(QueryBuilders.termQuery("k" + iterator, "hello" + iterator))
+                .get();
+            assertSearchResponse(resp);
+        }
+        requestCacheStats = getRequestCacheStats(client, "index");
+        // All entries should get cached.
+        assertEquals(numberOfIndexedItems * perQuerySizeInCacheInBytes, requestCacheStats.getMemorySizeInBytes());
+        // No more evictions seen when compared with last step.
+        assertEquals(0, requestCacheStats.getEvictions() - lastEvictionSeen);
+        // Hit count should be equal to number of cache entries present in previous step.
+        assertEquals(entriesInCache, requestCacheStats.getHitCount());
+        assertEquals(numberOfIndexedItems * 3 - entriesInCache, requestCacheStats.getMissCount());
+        long lastHitCountSeen = requestCacheStats.getHitCount();
+        long lastMissCountSeen = requestCacheStats.getMissCount();
+
+        // Step 4: Again hit the same requests, we should get hits for all entries.
+        for (int iterator = 0; iterator < numberOfIndexedItems; iterator++) {
+            SearchResponse resp = client.prepareSearch("index")
+                .setRequestCache(true)
+                .setQuery(QueryBuilders.termQuery("k" + iterator, "hello" + iterator))
+                .get();
+            assertSearchResponse(resp);
+        }
+        requestCacheStats = getRequestCacheStats(client, "index");
+        // All entries should get cached.
+        assertEquals(numberOfIndexedItems * perQuerySizeInCacheInBytes, requestCacheStats.getMemorySizeInBytes());
+        // No more evictions seen when compared with last step.
+        assertEquals(0, requestCacheStats.getEvictions() - lastEvictionSeen);
+        assertEquals(lastHitCountSeen + numberOfIndexedItems, requestCacheStats.getHitCount());
+        assertEquals(0, lastMissCountSeen - requestCacheStats.getMissCount());
+    }
+
+    public void testInvalidationWithIndicesRequestCache() throws Exception {
+        int onHeapCacheSizeInBytes = 2000;
+        internalCluster().startNode(
+            Settings.builder()
+                .put(defaultSettings(onHeapCacheSizeInBytes + "b"))
+                .put(INDICES_CACHE_CLEAN_INTERVAL_SETTING.getKey(), new TimeValue(1))
+                .build()
+        );
+        Client client = client();
+        assertAcked(
+            client.admin()
+                .indices()
+                .prepareCreate("index")
+                .setMapping("k", "type=keyword")
+                .setSettings(
+                    Settings.builder()
+                        .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
+                        .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                        .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                        .put("index.refresh_interval", -1)
+                )
+                .get()
+        );
+        // Update took time policy to zero so that all entries are eligible to be cached on disk.
+        ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest().transientSettings(
+            Settings.builder()
+                .put(
+                    TieredSpilloverCacheSettings.TOOK_TIME_POLICY_CONCRETE_SETTINGS_MAP.get(CacheType.INDICES_REQUEST_CACHE).getKey(),
+                    new TimeValue(0, TimeUnit.MILLISECONDS)
+                )
+                .build()
+        );
+        assertAcked(internalCluster().client().admin().cluster().updateSettings(updateSettingsRequest).get());
+        int numberOfIndexedItems = randomIntBetween(5, 10);
+        for (int iterator = 0; iterator < numberOfIndexedItems; iterator++) {
+            indexRandom(true, client.prepareIndex("index").setSource("k" + iterator, "hello" + iterator));
+        }
+        ensureSearchable("index");
+        refreshAndWaitForReplication();
+        // Force merge the index to ensure there can be no background merges during the subsequent searches that would invalidate the cache
+        ForceMergeResponse forceMergeResponse = client.admin().indices().prepareForceMerge("index").setFlush(true).get();
+        OpenSearchAssertions.assertAllSuccessful(forceMergeResponse);
+        long perQuerySizeInCacheInBytes = -1;
+        for (int iterator = 0; iterator < numberOfIndexedItems; iterator++) {
+            SearchResponse resp = client.prepareSearch("index")
+                .setRequestCache(true)
+                .setQuery(QueryBuilders.termQuery("k" + iterator, "hello" + iterator))
+                .get();
+            if (perQuerySizeInCacheInBytes == -1) {
+                RequestCacheStats requestCacheStats = getRequestCacheStats(client, "index");
+                perQuerySizeInCacheInBytes = requestCacheStats.getMemorySizeInBytes();
+            }
+            assertSearchResponse(resp);
+        }
+        RequestCacheStats requestCacheStats = getRequestCacheStats(client, "index");
+        assertEquals(numberOfIndexedItems, requestCacheStats.getMissCount());
+        assertEquals(0, requestCacheStats.getHitCount());
+        assertEquals(0, requestCacheStats.getEvictions());
+        assertEquals(perQuerySizeInCacheInBytes * numberOfIndexedItems, requestCacheStats.getMemorySizeInBytes());
+        for (int iterator = 0; iterator < numberOfIndexedItems; iterator++) {
+            SearchResponse resp = client.prepareSearch("index")
+                .setRequestCache(true)
+                .setQuery(QueryBuilders.termQuery("k" + iterator, "hello" + iterator))
+                .get();
+            assertSearchResponse(resp);
+        }
+        requestCacheStats = client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache();
+        assertEquals(numberOfIndexedItems, requestCacheStats.getHitCount());
+        assertEquals(numberOfIndexedItems, requestCacheStats.getMissCount());
+        assertEquals(perQuerySizeInCacheInBytes * numberOfIndexedItems, requestCacheStats.getMemorySizeInBytes());
+        assertEquals(0, requestCacheStats.getEvictions());
+        // Explicit refresh would invalidate cache entries.
+        refreshAndWaitForReplication();
+        assertBusy(() -> {
+            // Explicit refresh should clear up cache entries
+            assertTrue(getRequestCacheStats(client, "index").getMemorySizeInBytes() == 0);
+        }, 1, TimeUnit.SECONDS);
+        requestCacheStats = client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache();
+        assertEquals(0, requestCacheStats.getMemorySizeInBytes());
+        // Hits and misses stats shouldn't get cleared up.
+        assertEquals(numberOfIndexedItems, requestCacheStats.getHitCount());
+        assertEquals(numberOfIndexedItems, requestCacheStats.getMissCount());
+    }
+
+    public void testWithExplicitCacheClear() throws Exception {
+        int onHeapCacheSizeInBytes = 2000;
+        internalCluster().startNode(
+            Settings.builder()
+                .put(defaultSettings(onHeapCacheSizeInBytes + "b"))
+                .put(INDICES_CACHE_CLEAN_INTERVAL_SETTING.getKey(), new TimeValue(1))
+                .build()
+        );
+        Client client = client();
+        assertAcked(
+            client.admin()
+                .indices()
+                .prepareCreate("index")
+                .setMapping("k", "type=keyword")
+                .setSettings(
+                    Settings.builder()
+                        .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
+                        .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                        .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                        .put("index.refresh_interval", -1)
+                )
+                .get()
+        );
+        // Update took time policy to zero so that all entries are eligible to be cached on disk.
+        ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest().transientSettings(
+            Settings.builder()
+                .put(
+                    TieredSpilloverCacheSettings.TOOK_TIME_POLICY_CONCRETE_SETTINGS_MAP.get(CacheType.INDICES_REQUEST_CACHE).getKey(),
+                    new TimeValue(0, TimeUnit.MILLISECONDS)
+                )
+                .build()
+        );
+        assertAcked(internalCluster().client().admin().cluster().updateSettings(updateSettingsRequest).get());
+        int numberOfIndexedItems = randomIntBetween(5, 10);
+        for (int iterator = 0; iterator < numberOfIndexedItems; iterator++) {
+            indexRandom(true, client.prepareIndex("index").setSource("k" + iterator, "hello" + iterator));
+        }
+        ensureSearchable("index");
+        refreshAndWaitForReplication();
+        // Force merge the index to ensure there can be no background merges during the subsequent searches that would invalidate the cache
+        ForceMergeResponse forceMergeResponse = client.admin().indices().prepareForceMerge("index").setFlush(true).get();
+        OpenSearchAssertions.assertAllSuccessful(forceMergeResponse);
+
+        long perQuerySizeInCacheInBytes = -1;
+        for (int iterator = 0; iterator < numberOfIndexedItems; iterator++) {
+            SearchResponse resp = client.prepareSearch("index")
+                .setRequestCache(true)
+                .setQuery(QueryBuilders.termQuery("k" + iterator, "hello" + iterator))
+                .get();
+            if (perQuerySizeInCacheInBytes == -1) {
+                RequestCacheStats requestCacheStats = getRequestCacheStats(client, "index");
+                perQuerySizeInCacheInBytes = requestCacheStats.getMemorySizeInBytes();
+            }
+            assertSearchResponse(resp);
+        }
+        RequestCacheStats requestCacheStats = getRequestCacheStats(client, "index");
+        assertEquals(numberOfIndexedItems, requestCacheStats.getMissCount());
+        assertEquals(0, requestCacheStats.getHitCount());
+        assertEquals(0, requestCacheStats.getEvictions());
+        assertEquals(perQuerySizeInCacheInBytes * numberOfIndexedItems, requestCacheStats.getMemorySizeInBytes());
+
+        // Explicit clear the cache.
+        ClearIndicesCacheRequest request = new ClearIndicesCacheRequest("index");
+        ClearIndicesCacheResponse response = client.admin().indices().clearCache(request).get();
+        assertNoFailures(response);
+
+        assertBusy(() -> {
+            // All entries should get cleared up.
+            assertTrue(getRequestCacheStats(client, "index").getMemorySizeInBytes() == 0);
+        }, 1, TimeUnit.SECONDS);
+    }
+
+    public void testWithDynamicDiskCacheSetting() throws Exception {
+        int onHeapCacheSizeInBytes = 10; // Keep it low so that all items are cached onto disk.
+        internalCluster().startNode(
+            Settings.builder()
+                .put(defaultSettings(onHeapCacheSizeInBytes + "b"))
+                .put(INDICES_CACHE_CLEAN_INTERVAL_SETTING.getKey(), new TimeValue(1))
+                .build()
+        );
+        Client client = client();
+        assertAcked(
+            client.admin()
+                .indices()
+                .prepareCreate("index")
+                .setMapping("k", "type=keyword")
+                .setSettings(
+                    Settings.builder()
+                        .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
+                        .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                        .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                        .put("index.refresh_interval", -1)
+                )
+                .get()
+        );
+        // Update took time policy to zero so that all entries are eligible to be cached on disk.
+        ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest().transientSettings(
+            Settings.builder()
+                .put(
+                    TieredSpilloverCacheSettings.TOOK_TIME_POLICY_CONCRETE_SETTINGS_MAP.get(CacheType.INDICES_REQUEST_CACHE).getKey(),
+                    new TimeValue(0, TimeUnit.MILLISECONDS)
+                )
+                .build()
+        );
+        assertAcked(internalCluster().client().admin().cluster().updateSettings(updateSettingsRequest).get());
+        int numberOfIndexedItems = randomIntBetween(5, 10);
+        for (int iterator = 0; iterator < numberOfIndexedItems; iterator++) {
+            indexRandom(true, client.prepareIndex("index").setSource("k" + iterator, "hello" + iterator));
+        }
+        ensureSearchable("index");
+        refreshAndWaitForReplication();
+        // Force merge the index to ensure there can be no background merges during the subsequent searches that would invalidate the cache
+        ForceMergeResponse forceMergeResponse = client.admin().indices().prepareForceMerge("index").setFlush(true).get();
+        OpenSearchAssertions.assertAllSuccessful(forceMergeResponse);
+        long perQuerySizeInCacheInBytes = -1;
+        // Step 1: Hit some queries. We will see misses and queries will be cached(onto disk cache) for subsequent
+        // requests.
+        for (int iterator = 0; iterator < numberOfIndexedItems; iterator++) {
+            SearchResponse resp = client.prepareSearch("index")
+                .setRequestCache(true)
+                .setQuery(QueryBuilders.termQuery("k" + iterator, "hello" + iterator))
+                .get();
+            if (perQuerySizeInCacheInBytes == -1) {
+                RequestCacheStats requestCacheStats = getRequestCacheStats(client, "index");
+                perQuerySizeInCacheInBytes = requestCacheStats.getMemorySizeInBytes();
+            }
+            assertSearchResponse(resp);
+        }
+
+        RequestCacheStats requestCacheStats = getRequestCacheStats(client, "index");
+        assertEquals(numberOfIndexedItems * perQuerySizeInCacheInBytes, requestCacheStats.getMemorySizeInBytes());
+        assertEquals(numberOfIndexedItems, requestCacheStats.getMissCount());
+        assertEquals(0, requestCacheStats.getHitCount());
+        assertEquals(0, requestCacheStats.getEvictions());
+
+        // Step 2: Hit same queries again. We will see hits now.
+        for (int iterator = 0; iterator < numberOfIndexedItems; iterator++) {
+            SearchResponse resp = client.prepareSearch("index")
+                .setRequestCache(true)
+                .setQuery(QueryBuilders.termQuery("k" + iterator, "hello" + iterator))
+                .get();
+            assertSearchResponse(resp);
+        }
+        requestCacheStats = getRequestCacheStats(client, "index");
+        assertEquals(numberOfIndexedItems * perQuerySizeInCacheInBytes, requestCacheStats.getMemorySizeInBytes());
+        assertEquals(numberOfIndexedItems, requestCacheStats.getMissCount());
+        assertEquals(numberOfIndexedItems, requestCacheStats.getHitCount());
+        assertEquals(0, requestCacheStats.getEvictions());
+        long lastKnownHitCount = requestCacheStats.getHitCount();
+        long lastKnownMissCount = requestCacheStats.getMissCount();
+
+        // Step 3: Turn off disk cache now. And hit same queries again. We should not see hits now as all queries
+        // were cached onto disk cache.
+        updateSettingsRequest = new ClusterUpdateSettingsRequest().transientSettings(
+            Settings.builder()
+                .put(TieredSpilloverCacheSettings.DISK_CACHE_ENABLED_SETTING_MAP.get(CacheType.INDICES_REQUEST_CACHE).getKey(), false)
+                .build()
+        );
+        assertAcked(internalCluster().client().admin().cluster().updateSettings(updateSettingsRequest).get());
+
+        for (int iterator = 0; iterator < numberOfIndexedItems; iterator++) {
+            SearchResponse resp = client.prepareSearch("index")
+                .setRequestCache(true)
+                .setQuery(QueryBuilders.termQuery("k" + iterator, "hello" + iterator))
+                .get();
+            assertSearchResponse(resp);
+        }
+        requestCacheStats = getRequestCacheStats(client, "index");
+        assertEquals(numberOfIndexedItems * perQuerySizeInCacheInBytes, requestCacheStats.getMemorySizeInBytes()); //
+        // Still shows disk cache entries as explicit clear or invalidation is required to clean up disk cache.
+        assertEquals(lastKnownMissCount + numberOfIndexedItems, requestCacheStats.getMissCount());
+        assertEquals(0, lastKnownHitCount - requestCacheStats.getHitCount()); // No new hits being seen.
+        lastKnownMissCount = requestCacheStats.getMissCount();
+        lastKnownHitCount = requestCacheStats.getHitCount();
+
+        // Step 4: Invalidate entries via refresh.
+        // Explicit refresh would invalidate cache entries.
+        refreshAndWaitForReplication();
+        assertBusy(() -> {
+            // Explicit refresh should clear up cache entries
+            assertTrue(getRequestCacheStats(client, "index").getMemorySizeInBytes() == 0);
+        }, 1, TimeUnit.SECONDS);
+        requestCacheStats = getRequestCacheStats(client, "index");
+        assertEquals(0, lastKnownMissCount - requestCacheStats.getMissCount());
+        assertEquals(0, lastKnownHitCount - requestCacheStats.getHitCount());
+    }
+
+    private RequestCacheStats getRequestCacheStats(Client client, String indexName) {
+        return client.admin().indices().prepareStats(indexName).setRequestCache(true).get().getTotal().getRequestCache();
+    }
+
     public static class MockDiskCachePlugin extends Plugin implements CachePlugin {
 
         public MockDiskCachePlugin() {}
 
         @Override
         public Map<String, ICache.Factory> getCacheFactoryMap() {
-            return Map.of(MockDiskCache.MockDiskCacheFactory.NAME, new MockDiskCache.MockDiskCacheFactory(0, 1000));
+            return Map.of(MockDiskCache.MockDiskCacheFactory.NAME, new MockDiskCache.MockDiskCacheFactory(0, 1000, false));
         }
 
         @Override
diff --git a/modules/cache-common/src/main/java/org/opensearch/cache/common/policy/TookTimePolicy.java b/modules/cache-common/src/main/java/org/opensearch/cache/common/policy/TookTimePolicy.java
index 96ef027c17187..4bc26803acf4c 100644
--- a/modules/cache-common/src/main/java/org/opensearch/cache/common/policy/TookTimePolicy.java
+++ b/modules/cache-common/src/main/java/org/opensearch/cache/common/policy/TookTimePolicy.java
@@ -13,12 +13,16 @@
 
 package org.opensearch.cache.common.policy;
 
+import org.opensearch.common.cache.CacheType;
 import org.opensearch.common.cache.policy.CachedQueryResult;
+import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.unit.TimeValue;
 
 import java.util.function.Function;
 import java.util.function.Predicate;
 
+import static org.opensearch.cache.common.tier.TieredSpilloverCacheSettings.TOOK_TIME_POLICY_CONCRETE_SETTINGS_MAP;
+
 /**
  * A cache tier policy which accepts queries whose took time is greater than some threshold.
  * The threshold should be set to approximately the time it takes to get a result from the cache tier.
@@ -30,7 +34,7 @@ public class TookTimePolicy<V> implements Predicate<V> {
     /**
      * The minimum took time to allow a query. Set to TimeValue.ZERO to let all data through.
      */
-    private final TimeValue threshold;
+    private TimeValue threshold;
 
     /**
      * Function which extracts the relevant PolicyValues from a serialized CachedQueryResult
@@ -41,13 +45,25 @@ public class TookTimePolicy<V> implements Predicate<V> {
      * Constructs a took time policy.
      * @param threshold the threshold
      * @param cachedResultParser the function providing policy values
+     * @param clusterSettings cluster settings
+     * @param cacheType cache type
      */
-    public TookTimePolicy(TimeValue threshold, Function<V, CachedQueryResult.PolicyValues> cachedResultParser) {
+    public TookTimePolicy(
+        TimeValue threshold,
+        Function<V, CachedQueryResult.PolicyValues> cachedResultParser,
+        ClusterSettings clusterSettings,
+        CacheType cacheType
+    ) {
         if (threshold.compareTo(TimeValue.ZERO) < 0) {
             throw new IllegalArgumentException("Threshold for TookTimePolicy must be >= 0ms but was " + threshold.getStringRep());
         }
         this.threshold = threshold;
         this.cachedResultParser = cachedResultParser;
+        clusterSettings.addSettingsUpdateConsumer(TOOK_TIME_POLICY_CONCRETE_SETTINGS_MAP.get(cacheType), this::setThreshold);
+    }
+
+    private void setThreshold(TimeValue threshold) {
+        this.threshold = threshold;
     }
 
     /**
diff --git a/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCache.java b/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCache.java
index 00a8eec93acc9..9942651ccdd67 100644
--- a/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCache.java
+++ b/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCache.java
@@ -12,27 +12,39 @@
 import org.opensearch.common.annotation.ExperimentalApi;
 import org.opensearch.common.cache.CacheType;
 import org.opensearch.common.cache.ICache;
+import org.opensearch.common.cache.ICacheKey;
 import org.opensearch.common.cache.LoadAwareCacheLoader;
 import org.opensearch.common.cache.RemovalListener;
 import org.opensearch.common.cache.RemovalNotification;
+import org.opensearch.common.cache.RemovalReason;
 import org.opensearch.common.cache.policy.CachedQueryResult;
+import org.opensearch.common.cache.stats.ImmutableCacheStatsHolder;
 import org.opensearch.common.cache.store.config.CacheConfig;
+import org.opensearch.common.collect.Tuple;
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.util.concurrent.ReleasableLock;
-import org.opensearch.common.util.iterable.Iterables;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Arrays;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.NoSuchElementException;
 import java.util.Objects;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.locks.ReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.function.Function;
 import java.util.function.Predicate;
+import java.util.function.ToLongBiFunction;
+
+import static org.opensearch.cache.common.tier.TieredSpilloverCacheSettings.DISK_CACHE_ENABLED_SETTING_MAP;
+import static org.opensearch.cache.common.tier.TieredSpilloverCacheStatsHolder.TIER_DIMENSION_VALUE_DISK;
+import static org.opensearch.cache.common.tier.TieredSpilloverCacheStatsHolder.TIER_DIMENSION_VALUE_ON_HEAP;
 
 /**
  * This cache spillover the evicted items from heap tier to disk tier. All the new items are first cached on heap
@@ -47,49 +59,84 @@
 @ExperimentalApi
 public class TieredSpilloverCache<K, V> implements ICache<K, V> {
 
+    // Used to avoid caching stale entries in lower tiers.
+    private static final List<RemovalReason> SPILLOVER_REMOVAL_REASONS = List.of(RemovalReason.EVICTED, RemovalReason.CAPACITY);
+
     private final ICache<K, V> diskCache;
     private final ICache<K, V> onHeapCache;
-    private final RemovalListener<K, V> removalListener;
+
+    // Removal listeners for the individual tiers
+    private final RemovalListener<ICacheKey<K>, V> onDiskRemovalListener;
+    private final RemovalListener<ICacheKey<K>, V> onHeapRemovalListener;
+
+    // Removal listener from the spillover cache as a whole
+    private final RemovalListener<ICacheKey<K>, V> removalListener;
+
+    // In future we want to just read the stats from the individual tiers' statsHolder objects, but this isn't
+    // possible right now because of the way computeIfAbsent is implemented.
+    private final TieredSpilloverCacheStatsHolder statsHolder;
+    private ToLongBiFunction<ICacheKey<K>, V> weigher;
+    private final List<String> dimensionNames;
     ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
     ReleasableLock readLock = new ReleasableLock(readWriteLock.readLock());
     ReleasableLock writeLock = new ReleasableLock(readWriteLock.writeLock());
     /**
      * Maintains caching tiers in ascending order of cache latency.
      */
-    private final List<ICache<K, V>> cacheList;
+    private final Map<ICache<K, V>, TierInfo> caches;
     private final List<Predicate<V>> policies;
 
     TieredSpilloverCache(Builder<K, V> builder) {
         Objects.requireNonNull(builder.onHeapCacheFactory, "onHeap cache builder can't be null");
         Objects.requireNonNull(builder.diskCacheFactory, "disk cache builder can't be null");
+        Objects.requireNonNull(builder.cacheConfig, "cache config can't be null");
+        Objects.requireNonNull(builder.cacheConfig.getClusterSettings(), "cluster settings can't be null");
         this.removalListener = Objects.requireNonNull(builder.removalListener, "Removal listener can't be null");
 
+        this.onHeapRemovalListener = new HeapTierRemovalListener(this);
+        this.onDiskRemovalListener = new DiskTierRemovalListener(this);
+        this.weigher = Objects.requireNonNull(builder.cacheConfig.getWeigher(), "Weigher can't be null");
+
         this.onHeapCache = builder.onHeapCacheFactory.create(
-            new CacheConfig.Builder<K, V>().setRemovalListener(new RemovalListener<K, V>() {
-                @Override
-                public void onRemoval(RemovalNotification<K, V> notification) {
-                    try (ReleasableLock ignore = writeLock.acquire()) {
-                        if (evaluatePolicies(notification.getValue())) {
-                            diskCache.put(notification.getKey(), notification.getValue());
-                        }
-                    }
-                }
-            })
+            new CacheConfig.Builder<K, V>().setRemovalListener(onHeapRemovalListener)
                 .setKeyType(builder.cacheConfig.getKeyType())
                 .setValueType(builder.cacheConfig.getValueType())
                 .setSettings(builder.cacheConfig.getSettings())
                 .setWeigher(builder.cacheConfig.getWeigher())
+                .setDimensionNames(builder.cacheConfig.getDimensionNames())
                 .setMaxSizeInBytes(builder.cacheConfig.getMaxSizeInBytes())
                 .setExpireAfterAccess(builder.cacheConfig.getExpireAfterAccess())
+                .setClusterSettings(builder.cacheConfig.getClusterSettings())
+                .setStatsTrackingEnabled(false)
                 .build(),
             builder.cacheType,
             builder.cacheFactories
 
         );
-        this.diskCache = builder.diskCacheFactory.create(builder.cacheConfig, builder.cacheType, builder.cacheFactories);
-        this.cacheList = Arrays.asList(onHeapCache, diskCache);
-
+        this.diskCache = builder.diskCacheFactory.create(
+            new CacheConfig.Builder<K, V>().setRemovalListener(onDiskRemovalListener)
+                .setKeyType(builder.cacheConfig.getKeyType())
+                .setValueType(builder.cacheConfig.getValueType())
+                .setSettings(builder.cacheConfig.getSettings())
+                .setWeigher(builder.cacheConfig.getWeigher())
+                .setDimensionNames(builder.cacheConfig.getDimensionNames())
+                .setStatsTrackingEnabled(false)
+                .build(),
+            builder.cacheType,
+            builder.cacheFactories
+        );
+        Boolean isDiskCacheEnabled = DISK_CACHE_ENABLED_SETTING_MAP.get(builder.cacheType).get(builder.cacheConfig.getSettings());
+        LinkedHashMap<ICache<K, V>, TierInfo> cacheListMap = new LinkedHashMap<>();
+        cacheListMap.put(onHeapCache, new TierInfo(true, TIER_DIMENSION_VALUE_ON_HEAP));
+        cacheListMap.put(diskCache, new TierInfo(isDiskCacheEnabled, TIER_DIMENSION_VALUE_DISK));
+        this.caches = Collections.synchronizedMap(cacheListMap);
+
+        this.dimensionNames = builder.cacheConfig.getDimensionNames();
+        // Pass "tier" as the innermost dimension name, in addition to whatever dimensions are specified for the cache as a whole
+        this.statsHolder = new TieredSpilloverCacheStatsHolder(dimensionNames, isDiskCacheEnabled);
         this.policies = builder.policies; // Will never be null; builder initializes it to an empty list
+        builder.cacheConfig.getClusterSettings()
+            .addSettingsUpdateConsumer(DISK_CACHE_ENABLED_SETTING_MAP.get(builder.cacheType), this::enableDisableDiskCache);
     }
 
     // Package private for testing
@@ -102,23 +149,42 @@ ICache<K, V> getDiskCache() {
         return diskCache;
     }
 
+    // Package private for testing.
+    void enableDisableDiskCache(Boolean isDiskCacheEnabled) {
+        // When disk cache is disabled, we are not clearing up the disk cache entries yet as that should be part of
+        // separate cache/clear API.
+        this.caches.put(diskCache, new TierInfo(isDiskCacheEnabled, TIER_DIMENSION_VALUE_DISK));
+        this.statsHolder.setDiskCacheEnabled(isDiskCacheEnabled);
+    }
+
     @Override
-    public V get(K key) {
-        return getValueFromTieredCache().apply(key);
+    public V get(ICacheKey<K> key) {
+        Tuple<V, String> cacheValueTuple = getValueFromTieredCache(true).apply(key);
+        if (cacheValueTuple == null) {
+            return null;
+        }
+        return cacheValueTuple.v1();
     }
 
     @Override
-    public void put(K key, V value) {
+    public void put(ICacheKey<K> key, V value) {
         try (ReleasableLock ignore = writeLock.acquire()) {
             onHeapCache.put(key, value);
+            updateStatsOnPut(TIER_DIMENSION_VALUE_ON_HEAP, key, value);
         }
     }
 
     @Override
-    public V computeIfAbsent(K key, LoadAwareCacheLoader<K, V> loader) throws Exception {
-
-        V cacheValue = getValueFromTieredCache().apply(key);
-        if (cacheValue == null) {
+    public V computeIfAbsent(ICacheKey<K> key, LoadAwareCacheLoader<ICacheKey<K>, V> loader) throws Exception {
+        // Don't capture stats in the initial getValueFromTieredCache(). If we have concurrent requests for the same key,
+        // and it only has to be loaded one time, we should report one miss and the rest hits. But, if we do stats in
+        // getValueFromTieredCache(),
+        // we will see all misses. Instead, handle stats in computeIfAbsent().
+        Tuple<V, String> cacheValueTuple = getValueFromTieredCache(false).apply(key);
+        List<String> heapDimensionValues = statsHolder.getDimensionsWithTierValue(key.dimensions, TIER_DIMENSION_VALUE_ON_HEAP);
+        List<String> diskDimensionValues = statsHolder.getDimensionsWithTierValue(key.dimensions, TIER_DIMENSION_VALUE_DISK);
+
+        if (cacheValueTuple == null) {
             // Add the value to the onHeap cache. We are calling computeIfAbsent which does another get inside.
             // This is needed as there can be many requests for the same key at the same time and we only want to load
             // the value once.
@@ -126,19 +192,49 @@ public V computeIfAbsent(K key, LoadAwareCacheLoader<K, V> loader) throws Except
             try (ReleasableLock ignore = writeLock.acquire()) {
                 value = onHeapCache.computeIfAbsent(key, loader);
             }
+            // Handle stats
+            if (loader.isLoaded()) {
+                // The value was just computed and added to the cache by this thread. Register a miss for the heap cache, and the disk cache
+                // if present
+                updateStatsOnPut(TIER_DIMENSION_VALUE_ON_HEAP, key, value);
+                statsHolder.incrementMisses(heapDimensionValues);
+                if (caches.get(diskCache).isEnabled()) {
+                    statsHolder.incrementMisses(diskDimensionValues);
+                }
+            } else {
+                // Another thread requesting this key already loaded the value. Register a hit for the heap cache
+                statsHolder.incrementHits(heapDimensionValues);
+            }
             return value;
+        } else {
+            // Handle stats for an initial hit from getValueFromTieredCache()
+            if (cacheValueTuple.v2().equals(TIER_DIMENSION_VALUE_ON_HEAP)) {
+                // A hit for the heap tier
+                statsHolder.incrementHits(heapDimensionValues);
+            } else if (cacheValueTuple.v2().equals(TIER_DIMENSION_VALUE_DISK)) {
+                // Miss for the heap tier, hit for the disk tier
+                statsHolder.incrementMisses(heapDimensionValues);
+                statsHolder.incrementHits(diskDimensionValues);
+            }
         }
-        return cacheValue;
+        return cacheValueTuple.v1();
     }
 
     @Override
-    public void invalidate(K key) {
+    public void invalidate(ICacheKey<K> key) {
         // We are trying to invalidate the key from all caches though it would be present in only of them.
         // Doing this as we don't know where it is located. We could do a get from both and check that, but what will
         // also trigger a hit/miss listener event, so ignoring it for now.
-        try (ReleasableLock ignore = writeLock.acquire()) {
-            for (ICache<K, V> cache : cacheList) {
-                cache.invalidate(key);
+        // We don't update stats here, as this is handled by the removal listeners for the tiers.
+        for (Map.Entry<ICache<K, V>, TierInfo> cacheEntry : caches.entrySet()) {
+            if (key.getDropStatsForDimensions()) {
+                List<String> dimensionValues = statsHolder.getDimensionsWithTierValue(key.dimensions, cacheEntry.getValue().tierName);
+                statsHolder.removeDimensions(dimensionValues);
+            }
+            if (key.key != null) {
+                try (ReleasableLock ignore = writeLock.acquire()) {
+                    cacheEntry.getKey().invalidate(key);
+                }
             }
         }
     }
@@ -146,64 +242,124 @@ public void invalidate(K key) {
     @Override
     public void invalidateAll() {
         try (ReleasableLock ignore = writeLock.acquire()) {
-            for (ICache<K, V> cache : cacheList) {
-                cache.invalidateAll();
+            for (Map.Entry<ICache<K, V>, TierInfo> cacheEntry : caches.entrySet()) {
+                cacheEntry.getKey().invalidateAll();
             }
         }
+        statsHolder.reset();
     }
 
     /**
      * Provides an iteration over both onHeap and disk keys. This is not protected from any mutations to the cache.
      * @return An iterable over (onHeap + disk) keys
      */
-    @SuppressWarnings("unchecked")
+    @SuppressWarnings({ "unchecked" })
     @Override
-    public Iterable<K> keys() {
-        return Iterables.concat(onHeapCache.keys(), diskCache.keys());
+    public Iterable<ICacheKey<K>> keys() {
+        List<Iterable<ICacheKey<K>>> iterableList = new ArrayList<>();
+        for (Map.Entry<ICache<K, V>, TierInfo> cacheEntry : caches.entrySet()) {
+            iterableList.add(cacheEntry.getKey().keys());
+        }
+        Iterable<ICacheKey<K>>[] iterables = (Iterable<ICacheKey<K>>[]) iterableList.toArray(new Iterable<?>[0]);
+        return new ConcatenatedIterables<>(iterables);
     }
 
     @Override
     public long count() {
-        long count = 0;
-        for (ICache<K, V> cache : cacheList) {
-            count += cache.count();
-        }
-        return count;
+        // Count for all the tiers irrespective of whether they are enabled or not. As eventually
+        // this will turn to zero once cache is cleared up either via invalidation or manually.
+        return statsHolder.count();
     }
 
     @Override
     public void refresh() {
         try (ReleasableLock ignore = writeLock.acquire()) {
-            for (ICache<K, V> cache : cacheList) {
-                cache.refresh();
+            for (Map.Entry<ICache<K, V>, TierInfo> cacheEntry : caches.entrySet()) {
+                cacheEntry.getKey().refresh();
             }
         }
     }
 
     @Override
     public void close() throws IOException {
-        for (ICache<K, V> cache : cacheList) {
-            cache.close();
+        for (Map.Entry<ICache<K, V>, TierInfo> cacheEntry : caches.entrySet()) {
+            // Close all the caches here irrespective of whether they are enabled or not.
+            cacheEntry.getKey().close();
         }
     }
 
-    private Function<K, V> getValueFromTieredCache() {
+    @Override
+    public ImmutableCacheStatsHolder stats(String[] levels) {
+        return statsHolder.getImmutableCacheStatsHolder(levels);
+    }
+
+    /**
+     * Get a value from the tiered cache, and the name of the tier it was found in.
+     * @param captureStats Whether to record hits/misses for this call of the function
+     * @return A tuple of the value and the name of the tier it was found in.
+     */
+    private Function<ICacheKey<K>, Tuple<V, String>> getValueFromTieredCache(boolean captureStats) {
         return key -> {
             try (ReleasableLock ignore = readLock.acquire()) {
-                for (ICache<K, V> cache : cacheList) {
-                    V value = cache.get(key);
-                    if (value != null) {
-                        // update hit stats
-                        return value;
-                    } else {
-                        // update miss stats
+                for (Map.Entry<ICache<K, V>, TierInfo> cacheEntry : caches.entrySet()) {
+                    if (cacheEntry.getValue().isEnabled()) {
+                        V value = cacheEntry.getKey().get(key);
+                        // Get the tier value corresponding to this cache
+                        String tierValue = cacheEntry.getValue().tierName;
+                        List<String> dimensionValues = statsHolder.getDimensionsWithTierValue(key.dimensions, tierValue);
+                        if (value != null) {
+                            if (captureStats) {
+                                statsHolder.incrementHits(dimensionValues);
+                            }
+                            return new Tuple<>(value, tierValue);
+                        } else if (captureStats) {
+                            statsHolder.incrementMisses(dimensionValues);
+                        }
                     }
                 }
+                return null;
             }
-            return null;
         };
     }
 
+    void handleRemovalFromHeapTier(RemovalNotification<ICacheKey<K>, V> notification) {
+        ICacheKey<K> key = notification.getKey();
+        boolean wasEvicted = SPILLOVER_REMOVAL_REASONS.contains(notification.getRemovalReason());
+        if (caches.get(diskCache).isEnabled() && wasEvicted && evaluatePolicies(notification.getValue())) {
+            try (ReleasableLock ignore = writeLock.acquire()) {
+                diskCache.put(key, notification.getValue()); // spill over to the disk tier and increment its stats
+            }
+            updateStatsOnPut(TIER_DIMENSION_VALUE_DISK, key, notification.getValue());
+        } else {
+            // If the value is not going to the disk cache, send this notification to the TSC's removal listener
+            // as the value is leaving the TSC entirely
+            removalListener.onRemoval(notification);
+        }
+        updateStatsOnRemoval(TIER_DIMENSION_VALUE_ON_HEAP, wasEvicted, key, notification.getValue());
+    }
+
+    void handleRemovalFromDiskTier(RemovalNotification<ICacheKey<K>, V> notification) {
+        // Values removed from the disk tier leave the TSC entirely
+        removalListener.onRemoval(notification);
+        boolean wasEvicted = SPILLOVER_REMOVAL_REASONS.contains(notification.getRemovalReason());
+        updateStatsOnRemoval(TIER_DIMENSION_VALUE_DISK, wasEvicted, notification.getKey(), notification.getValue());
+    }
+
+    void updateStatsOnRemoval(String removedFromTierValue, boolean wasEvicted, ICacheKey<K> key, V value) {
+        List<String> dimensionValues = statsHolder.getDimensionsWithTierValue(key.dimensions, removedFromTierValue);
+        if (wasEvicted) {
+            statsHolder.incrementEvictions(dimensionValues);
+        }
+        statsHolder.decrementItems(dimensionValues);
+        statsHolder.decrementSizeInBytes(dimensionValues, weigher.applyAsLong(key, value));
+    }
+
+    void updateStatsOnPut(String destinationTierValue, ICacheKey<K> key, V value) {
+        List<String> dimensionValues = statsHolder.getDimensionsWithTierValue(key.dimensions, destinationTierValue);
+        statsHolder.incrementItems(dimensionValues);
+        statsHolder.incrementSizeInBytes(dimensionValues, weigher.applyAsLong(key, value));
+    }
+
     boolean evaluatePolicies(V value) {
         for (Predicate<V> policy : policies) {
             if (!policy.test(value)) {
@@ -213,6 +369,113 @@ boolean evaluatePolicies(V value) {
         return true;
     }
 
+    /**
+     * A class which receives removal events from the heap tier.
+     */
+    private class HeapTierRemovalListener implements RemovalListener<ICacheKey<K>, V> {
+        private final TieredSpilloverCache<K, V> tsc;
+
+        HeapTierRemovalListener(TieredSpilloverCache<K, V> tsc) {
+            this.tsc = tsc;
+        }
+
+        @Override
+        public void onRemoval(RemovalNotification<ICacheKey<K>, V> notification) {
+            tsc.handleRemovalFromHeapTier(notification);
+        }
+    }
+
+    /**
+     * A class which receives removal events from the disk tier.
+     */
+    private class DiskTierRemovalListener implements RemovalListener<ICacheKey<K>, V> {
+        private final TieredSpilloverCache<K, V> tsc;
+
+        DiskTierRemovalListener(TieredSpilloverCache<K, V> tsc) {
+            this.tsc = tsc;
+        }
+
+        @Override
+        public void onRemoval(RemovalNotification<ICacheKey<K>, V> notification) {
+            tsc.handleRemovalFromDiskTier(notification);
+        }
+    }
+
+    /**
+     * ConcatenatedIterables which combines cache iterables and supports remove() functionality as well if underlying
+     * iterator supports it.
+     * @param <K> Type of key.
+     */
+    static class ConcatenatedIterables<K> implements Iterable<K> {
+
+        final Iterable<K>[] iterables;
+
+        ConcatenatedIterables(Iterable<K>[] iterables) {
+            this.iterables = iterables;
+        }
+
+        @SuppressWarnings({ "unchecked" })
+        @Override
+        public Iterator<K> iterator() {
+            Iterator<K>[] iterators = (Iterator<K>[]) new Iterator<?>[iterables.length];
+            for (int i = 0; i < iterables.length; i++) {
+                iterators[i] = iterables[i].iterator();
+            }
+            return new ConcatenatedIterator<>(iterators);
+        }
+
+        static class ConcatenatedIterator<T> implements Iterator<T> {
+            private final Iterator<T>[] iterators;
+            private int currentIteratorIndex;
+            private Iterator<T> currentIterator;
+
+            public ConcatenatedIterator(Iterator<T>[] iterators) {
+                this.iterators = iterators;
+                this.currentIteratorIndex = 0;
+                this.currentIterator = iterators[currentIteratorIndex];
+            }
+
+            @Override
+            public boolean hasNext() {
+                while (!currentIterator.hasNext()) {
+                    currentIteratorIndex++;
+                    if (currentIteratorIndex == iterators.length) {
+                        return false;
+                    }
+                    currentIterator = iterators[currentIteratorIndex];
+                }
+                return true;
+            }
+
+            @Override
+            public T next() {
+                if (!hasNext()) {
+                    throw new NoSuchElementException();
+                }
+                return currentIterator.next();
+            }
+
+            @Override
+            public void remove() {
+                currentIterator.remove();
+            }
+        }
+    }
+
+    private class TierInfo {
+        AtomicBoolean isEnabled;
+        final String tierName;
+
+        TierInfo(boolean isEnabled, String tierName) {
+            this.isEnabled = new AtomicBoolean(isEnabled);
+            this.tierName = tierName;
+        }
+
+        boolean isEnabled() {
+            return isEnabled.get();
+        }
+    }
+
     /**
      * Factory to create TieredSpilloverCache objects.
      */
@@ -253,8 +516,7 @@ public <K, V> ICache<K, V> create(CacheConfig<K, V> config, CacheType cacheType,
             }
             ICache.Factory diskCacheFactory = cacheFactories.get(diskCacheStoreName);
 
-            TimeValue diskPolicyThreshold = TieredSpilloverCacheSettings.TIERED_SPILLOVER_DISK_TOOK_TIME_THRESHOLD
-                .getConcreteSettingForNamespace(cacheType.getSettingPrefix())
+            TimeValue diskPolicyThreshold = TieredSpilloverCacheSettings.TOOK_TIME_POLICY_CONCRETE_SETTINGS_MAP.get(cacheType)
                 .get(settings);
             Function<V, CachedQueryResult.PolicyValues> cachedResultParser = Objects.requireNonNull(
                 config.getCachedResultParser(),
@@ -266,7 +528,7 @@ public <K, V> ICache<K, V> create(CacheConfig<K, V> config, CacheType cacheType,
                 .setRemovalListener(config.getRemovalListener())
                 .setCacheConfig(config)
                 .setCacheType(cacheType)
-                .addPolicy(new TookTimePolicy<V>(diskPolicyThreshold, cachedResultParser))
+                .addPolicy(new TookTimePolicy<V>(diskPolicyThreshold, cachedResultParser, config.getClusterSettings(), cacheType))
                 .build();
         }
 
@@ -284,7 +546,7 @@ public String getCacheName() {
     public static class Builder<K, V> {
         private ICache.Factory onHeapCacheFactory;
         private ICache.Factory diskCacheFactory;
-        private RemovalListener<K, V> removalListener;
+        private RemovalListener<ICacheKey<K>, V> removalListener;
         private CacheConfig<K, V> cacheConfig;
         private CacheType cacheType;
         private Map<String, ICache.Factory> cacheFactories;
@@ -320,7 +582,7 @@ public Builder<K, V> setDiskCacheFactory(ICache.Factory diskCacheFactory) {
          * @param removalListener Removal listener
          * @return builder
          */
-        public Builder<K, V> setRemovalListener(RemovalListener<K, V> removalListener) {
+        public Builder<K, V> setRemovalListener(RemovalListener<ICacheKey<K>, V> removalListener) {
             this.removalListener = removalListener;
             return this;
         }
diff --git a/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCachePlugin.java b/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCachePlugin.java
index 0cc8a711faaf5..1c10e51630460 100644
--- a/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCachePlugin.java
+++ b/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCachePlugin.java
@@ -11,6 +11,8 @@
 import org.opensearch.common.cache.CacheType;
 import org.opensearch.common.cache.ICache;
 import org.opensearch.common.settings.Setting;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.plugins.CachePlugin;
 import org.opensearch.plugins.Plugin;
 
@@ -18,6 +20,9 @@
 import java.util.List;
 import java.util.Map;
 
+import static org.opensearch.cache.common.tier.TieredSpilloverCacheSettings.DISK_CACHE_ENABLED_SETTING_MAP;
+import static org.opensearch.cache.common.tier.TieredSpilloverCacheSettings.TOOK_TIME_POLICY_CONCRETE_SETTINGS_MAP;
+
 /**
  * Plugin for TieredSpilloverCache.
  */
@@ -28,10 +33,15 @@ public class TieredSpilloverCachePlugin extends Plugin implements CachePlugin {
      */
     public static final String TIERED_CACHE_SPILLOVER_PLUGIN_NAME = "tieredSpilloverCachePlugin";
 
+    private final Settings settings;
+
     /**
      * Default constructor
+     * @param settings settings
      */
-    public TieredSpilloverCachePlugin() {}
+    public TieredSpilloverCachePlugin(Settings settings) {
+        this.settings = settings;
+    }
 
     @Override
     public Map<String, ICache.Factory> getCacheFactoryMap() {
@@ -51,11 +61,10 @@ public List<Setting<?>> getSettings() {
             settingList.add(
                 TieredSpilloverCacheSettings.TIERED_SPILLOVER_DISK_STORE_NAME.getConcreteSettingForNamespace(cacheType.getSettingPrefix())
             );
-            settingList.add(
-                TieredSpilloverCacheSettings.TIERED_SPILLOVER_DISK_TOOK_TIME_THRESHOLD.getConcreteSettingForNamespace(
-                    cacheType.getSettingPrefix()
-                )
-            );
+            settingList.add(TOOK_TIME_POLICY_CONCRETE_SETTINGS_MAP.get(cacheType));
+            if (FeatureFlags.PLUGGABLE_CACHE_SETTING.get(settings)) {
+                settingList.add(DISK_CACHE_ENABLED_SETTING_MAP.get(cacheType));
+            }
         }
         return settingList;
     }
diff --git a/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCacheSettings.java b/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCacheSettings.java
index 684307960b8a5..e8e441d6bd3a6 100644
--- a/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCacheSettings.java
+++ b/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCacheSettings.java
@@ -8,9 +8,12 @@
 
 package org.opensearch.cache.common.tier;
 
+import org.opensearch.common.cache.CacheType;
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.unit.TimeValue;
 
+import java.util.HashMap;
+import java.util.Map;
 import java.util.concurrent.TimeUnit;
 
 import static org.opensearch.common.settings.Setting.Property.NodeScope;
@@ -39,20 +42,59 @@ public class TieredSpilloverCacheSettings {
         (key) -> Setting.simpleString(key, "", NodeScope)
     );
 
+    /**
+     * Setting to disable/enable disk cache dynamically.
+     */
+    public static final Setting.AffixSetting<Boolean> TIERED_SPILLOVER_DISK_CACHE_SETTING = Setting.suffixKeySetting(
+        TieredSpilloverCache.TieredSpilloverCacheFactory.TIERED_SPILLOVER_CACHE_NAME + ".disk.store.enabled",
+        (key) -> Setting.boolSetting(key, true, NodeScope, Setting.Property.Dynamic)
+    );
+
     /**
      * Setting defining the minimum took time for a query to be allowed into the disk cache.
      */
-    public static final Setting.AffixSetting<TimeValue> TIERED_SPILLOVER_DISK_TOOK_TIME_THRESHOLD = Setting.suffixKeySetting(
+    private static final Setting.AffixSetting<TimeValue> TIERED_SPILLOVER_DISK_TOOK_TIME_THRESHOLD = Setting.suffixKeySetting(
         TieredSpilloverCache.TieredSpilloverCacheFactory.TIERED_SPILLOVER_CACHE_NAME + ".disk.store.policies.took_time.threshold",
         (key) -> Setting.timeSetting(
             key,
             new TimeValue(10, TimeUnit.MILLISECONDS), // Default value for this setting
             TimeValue.ZERO, // Minimum value for this setting
-            NodeScope
+            NodeScope,
+            Setting.Property.Dynamic
         )
     );
-    // 10 ms was chosen as a safe value based on proof of concept, where we saw disk latencies in this range.
-    // Will be tuned further with future benchmarks.
+
+    /**
+     * Stores took time policy settings for various cache types as these are dynamic so that can be registered and
+     * retrieved accordingly.
+     */
+    public static final Map<CacheType, Setting<TimeValue>> TOOK_TIME_POLICY_CONCRETE_SETTINGS_MAP;
+
+    /**
+     * Stores disk cache enabled settings for various cache types as these are dynamic so that can be registered and
+     * retrieved accordingly.
+     */
+    public static final Map<CacheType, Setting<Boolean>> DISK_CACHE_ENABLED_SETTING_MAP;
+
+    /**
+     * Fetches concrete took time policy and disk cache settings.
+     */
+    static {
+        Map<CacheType, Setting<TimeValue>> concreteTookTimePolicySettingMap = new HashMap<>();
+        Map<CacheType, Setting<Boolean>> diskCacheSettingMap = new HashMap<>();
+        for (CacheType cacheType : CacheType.values()) {
+            concreteTookTimePolicySettingMap.put(
+                cacheType,
+                TIERED_SPILLOVER_DISK_TOOK_TIME_THRESHOLD.getConcreteSettingForNamespace(cacheType.getSettingPrefix())
+            );
+            diskCacheSettingMap.put(
+                cacheType,
+                TIERED_SPILLOVER_DISK_CACHE_SETTING.getConcreteSettingForNamespace(cacheType.getSettingPrefix())
+            );
+        }
+        TOOK_TIME_POLICY_CONCRETE_SETTINGS_MAP = concreteTookTimePolicySettingMap;
+        DISK_CACHE_ENABLED_SETTING_MAP = diskCacheSettingMap;
+    }
 
     /**
      * Default constructor
diff --git a/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCacheStatsHolder.java b/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCacheStatsHolder.java
new file mode 100644
index 0000000000000..d17059e8dee94
--- /dev/null
+++ b/modules/cache-common/src/main/java/org/opensearch/cache/common/tier/TieredSpilloverCacheStatsHolder.java
@@ -0,0 +1,161 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.cache.common.tier;
+
+import org.opensearch.common.cache.stats.DefaultCacheStatsHolder;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Consumer;
+
+/**
+ * A tier-aware version of DefaultCacheStatsHolder. Overrides the incrementer functions, as we can't just add the on-heap
+ * and disk stats to get a total for the cache as a whole. If the disk tier is present, the total hits, size, and entries
+ * should be the sum of both tiers' values, but the total misses and evictions should be the disk tier's values.
+ * When the disk tier isn't present, on-heap misses and evictions should contribute to the total.
+ *
+ * For example, if the heap tier has 5 misses and the disk tier has 4, the total cache has had 4 misses, not 9.
+ * The same goes for evictions. Other stats values add normally.
+ *
+ * This means for misses and evictions, if we are incrementing for the on-heap tier and the disk tier is present,
+ * we have to increment only the leaf nodes corresponding to the on-heap tier itself, and not its ancestors,
+ * which correspond to totals including both tiers. If the disk tier is not present, we do increment the ancestor nodes.
+ */
+public class TieredSpilloverCacheStatsHolder extends DefaultCacheStatsHolder {
+
+    /** Whether the disk cache is currently enabled. */
+    private boolean diskCacheEnabled;
+
+    // Common values used for tier dimension
+
+    /** The name for the tier dimension. */
+    public static final String TIER_DIMENSION_NAME = "tier";
+
+    /** Dimension value for on-heap cache, like OpenSearchOnHeapCache.*/
+    public static final String TIER_DIMENSION_VALUE_ON_HEAP = "on_heap";
+
+    /** Dimension value for on-disk cache, like EhcacheDiskCache. */
+    public static final String TIER_DIMENSION_VALUE_DISK = "disk";
+
+    /**
+     * Constructor for the stats holder.
+     * @param originalDimensionNames the original dimension names, not including TIER_DIMENSION_NAME
+     * @param diskCacheEnabled whether the disk tier starts out enabled
+     */
+    public TieredSpilloverCacheStatsHolder(List<String> originalDimensionNames, boolean diskCacheEnabled) {
+        super(
+            getDimensionNamesWithTier(originalDimensionNames),
+            TieredSpilloverCache.TieredSpilloverCacheFactory.TIERED_SPILLOVER_CACHE_NAME
+        );
+        this.diskCacheEnabled = diskCacheEnabled;
+    }
+
+    private static List<String> getDimensionNamesWithTier(List<String> dimensionNames) {
+        List<String> dimensionNamesWithTier = new ArrayList<>(dimensionNames);
+        dimensionNamesWithTier.add(TIER_DIMENSION_NAME);
+        return dimensionNamesWithTier;
+    }
+
+    /**
+     * Add tierValue to the end of a copy of the initial dimension values, so they can appropriately be used in this stats holder.
+     */
+    List<String> getDimensionsWithTierValue(List<String> initialDimensions, String tierValue) {
+        List<String> result = new ArrayList<>(initialDimensions);
+        result.add(tierValue);
+        return result;
+    }
+
+    private String validateTierDimensionValue(List<String> dimensionValues) {
+        String tierDimensionValue = dimensionValues.get(dimensionValues.size() - 1);
+        assert tierDimensionValue.equals(TIER_DIMENSION_VALUE_ON_HEAP) || tierDimensionValue.equals(TIER_DIMENSION_VALUE_DISK)
+            : "Invalid tier dimension value";
+        return tierDimensionValue;
+    }
+
+    @Override
+    public void incrementHits(List<String> dimensionValues) {
+        validateTierDimensionValue(dimensionValues);
+        // Hits from either tier should be included in the total values.
+        super.incrementHits(dimensionValues);
+    }
+
+    @Override
+    public void incrementMisses(List<String> dimensionValues) {
+        final String tierValue = validateTierDimensionValue(dimensionValues);
+
+        // If the disk tier is present, only misses from the disk tier should be included in total values.
+        Consumer<Node> missIncrementer = (node) -> {
+            if (tierValue.equals(TIER_DIMENSION_VALUE_ON_HEAP) && diskCacheEnabled) {
+                // If on-heap tier, increment only the leaf node corresponding to the on heap values; not the total values in its parent
+                // nodes
+                if (node.isAtLowestLevel()) {
+                    node.incrementMisses();
+                }
+            } else {
+                // If disk tier, or on-heap tier with a disabled disk tier, increment the leaf node and its parents
+                node.incrementMisses();
+            }
+        };
+        internalIncrement(dimensionValues, missIncrementer, true);
+    }
+
+    @Override
+    public void incrementEvictions(List<String> dimensionValues) {
+        final String tierValue = validateTierDimensionValue(dimensionValues);
+
+        // If the disk tier is present, only evictions from the disk tier should be included in total values.
+        Consumer<DefaultCacheStatsHolder.Node> evictionsIncrementer = (node) -> {
+            if (tierValue.equals(TIER_DIMENSION_VALUE_ON_HEAP) && diskCacheEnabled) {
+                // If on-heap tier, increment only the leaf node corresponding to the on heap values; not the total values in its parent
+                // nodes
+                if (node.isAtLowestLevel()) {
+                    node.incrementEvictions();
+                }
+            } else {
+                // If disk tier, or on-heap tier with a disabled disk tier, increment the leaf node and its parents
+                node.incrementEvictions();
+            }
+        };
+        internalIncrement(dimensionValues, evictionsIncrementer, true);
+    }
+
+    @Override
+    public void incrementSizeInBytes(List<String> dimensionValues, long amountBytes) {
+        validateTierDimensionValue(dimensionValues);
+        // Size from either tier should be included in the total values.
+        super.incrementSizeInBytes(dimensionValues, amountBytes);
+    }
+
+    // For decrements, we should not create nodes if they are absent. This protects us from erroneously decrementing values for keys
+    // which have been entirely deleted, for example in an async removal listener.
+    @Override
+    public void decrementSizeInBytes(List<String> dimensionValues, long amountBytes) {
+        validateTierDimensionValue(dimensionValues);
+        // Size from either tier should be included in the total values.
+        super.decrementSizeInBytes(dimensionValues, amountBytes);
+    }
+
+    @Override
+    public void incrementItems(List<String> dimensionValues) {
+        validateTierDimensionValue(dimensionValues);
+        // Entries from either tier should be included in the total values.
+        super.incrementItems(dimensionValues);
+    }
+
+    @Override
+    public void decrementItems(List<String> dimensionValues) {
+        validateTierDimensionValue(dimensionValues);
+        // Entries from either tier should be included in the total values.
+        super.decrementItems(dimensionValues);
+    }
+
+    void setDiskCacheEnabled(boolean diskCacheEnabled) {
+        this.diskCacheEnabled = diskCacheEnabled;
+    }
+}
diff --git a/modules/cache-common/src/test/java/org/opensearch/cache/common/policy/TookTimePolicyTests.java b/modules/cache-common/src/test/java/org/opensearch/cache/common/policy/TookTimePolicyTests.java
index 237c9c7b79db4..000067280e50d 100644
--- a/modules/cache-common/src/test/java/org/opensearch/cache/common/policy/TookTimePolicyTests.java
+++ b/modules/cache-common/src/test/java/org/opensearch/cache/common/policy/TookTimePolicyTests.java
@@ -12,20 +12,27 @@
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.search.TotalHits;
 import org.opensearch.common.Randomness;
+import org.opensearch.common.cache.CacheType;
 import org.opensearch.common.cache.policy.CachedQueryResult;
 import org.opensearch.common.io.stream.BytesStreamOutput;
 import org.opensearch.common.lucene.search.TopDocsAndMaxScore;
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.search.DocValueFormat;
 import org.opensearch.search.query.QuerySearchResult;
 import org.opensearch.test.OpenSearchTestCase;
+import org.junit.Before;
 
 import java.io.IOException;
+import java.util.HashSet;
 import java.util.Random;
 import java.util.function.Function;
 
+import static org.opensearch.cache.common.tier.TieredSpilloverCacheSettings.TOOK_TIME_POLICY_CONCRETE_SETTINGS_MAP;
+
 public class TookTimePolicyTests extends OpenSearchTestCase {
     private final Function<BytesReference, CachedQueryResult.PolicyValues> transformationFunction = (data) -> {
         try {
@@ -35,8 +42,17 @@ public class TookTimePolicyTests extends OpenSearchTestCase {
         }
     };
 
+    private ClusterSettings clusterSettings;
+
+    @Before
+    public void setup() {
+        Settings settings = Settings.EMPTY;
+        clusterSettings = new ClusterSettings(settings, new HashSet<>());
+        clusterSettings.registerSetting(TOOK_TIME_POLICY_CONCRETE_SETTINGS_MAP.get(CacheType.INDICES_REQUEST_CACHE));
+    }
+
     private TookTimePolicy<BytesReference> getTookTimePolicy(TimeValue threshold) {
-        return new TookTimePolicy<>(threshold, transformationFunction);
+        return new TookTimePolicy<>(threshold, transformationFunction, clusterSettings, CacheType.INDICES_REQUEST_CACHE);
     }
 
     public void testTookTimePolicy() throws Exception {
diff --git a/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/MockDiskCache.java b/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/MockDiskCache.java
index d8a6eb480a5a5..2058faa5181b1 100644
--- a/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/MockDiskCache.java
+++ b/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/MockDiskCache.java
@@ -10,42 +10,58 @@
 
 import org.opensearch.common.cache.CacheType;
 import org.opensearch.common.cache.ICache;
+import org.opensearch.common.cache.ICacheKey;
 import org.opensearch.common.cache.LoadAwareCacheLoader;
 import org.opensearch.common.cache.RemovalListener;
 import org.opensearch.common.cache.RemovalNotification;
 import org.opensearch.common.cache.RemovalReason;
 import org.opensearch.common.cache.serializer.Serializer;
+import org.opensearch.common.cache.stats.CacheStatsHolder;
+import org.opensearch.common.cache.stats.DefaultCacheStatsHolder;
+import org.opensearch.common.cache.stats.ImmutableCacheStatsHolder;
+import org.opensearch.common.cache.stats.NoopCacheStatsHolder;
 import org.opensearch.common.cache.store.builders.ICacheBuilder;
 import org.opensearch.common.cache.store.config.CacheConfig;
 
+import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
+import java.util.NoSuchElementException;
 import java.util.concurrent.ConcurrentHashMap;
 
 public class MockDiskCache<K, V> implements ICache<K, V> {
 
-    Map<K, V> cache;
+    Map<ICacheKey<K>, V> cache;
     int maxSize;
     long delay;
 
-    private final RemovalListener<K, V> removalListener;
+    private final RemovalListener<ICacheKey<K>, V> removalListener;
+    private final CacheStatsHolder statsHolder; // Only update for number of entries; this is only used to test statsTrackingEnabled logic
+                                                // in TSC
 
-    public MockDiskCache(int maxSize, long delay, RemovalListener<K, V> removalListener) {
+    public MockDiskCache(int maxSize, long delay, RemovalListener<ICacheKey<K>, V> removalListener, boolean statsTrackingEnabled) {
         this.maxSize = maxSize;
         this.delay = delay;
         this.removalListener = removalListener;
-        this.cache = new ConcurrentHashMap<K, V>();
+        this.cache = new ConcurrentHashMap<ICacheKey<K>, V>();
+        if (statsTrackingEnabled) {
+            this.statsHolder = new DefaultCacheStatsHolder(List.of(), "mock_disk_cache");
+        } else {
+            this.statsHolder = NoopCacheStatsHolder.getInstance();
+        }
     }
 
     @Override
-    public V get(K key) {
+    public V get(ICacheKey<K> key) {
         V value = cache.get(key);
         return value;
     }
 
     @Override
-    public void put(K key, V value) {
+    public void put(ICacheKey<K> key, V value) {
         if (this.cache.size() >= maxSize) { // For simplification
             this.removalListener.onRemoval(new RemovalNotification<>(key, value, RemovalReason.EVICTED));
+            this.statsHolder.decrementItems(List.of());
         }
         try {
             Thread.sleep(delay);
@@ -53,10 +69,11 @@ public void put(K key, V value) {
             throw new RuntimeException(e);
         }
         this.cache.put(key, value);
+        this.statsHolder.incrementItems(List.of());
     }
 
     @Override
-    public V computeIfAbsent(K key, LoadAwareCacheLoader<K, V> loader) {
+    public V computeIfAbsent(ICacheKey<K> key, LoadAwareCacheLoader<ICacheKey<K>, V> loader) {
         V value = cache.computeIfAbsent(key, key1 -> {
             try {
                 return loader.load(key);
@@ -68,7 +85,8 @@ public V computeIfAbsent(K key, LoadAwareCacheLoader<K, V> loader) {
     }
 
     @Override
-    public void invalidate(K key) {
+    public void invalidate(ICacheKey<K> key) {
+        removalListener.onRemoval(new RemovalNotification<>(key, cache.get(key), RemovalReason.INVALIDATED));
         this.cache.remove(key);
     }
 
@@ -78,8 +96,8 @@ public void invalidateAll() {
     }
 
     @Override
-    public Iterable<K> keys() {
-        return this.cache.keySet();
+    public Iterable<ICacheKey<K>> keys() {
+        return () -> new CacheKeyIterator<>(cache, removalListener);
     }
 
     @Override
@@ -90,6 +108,18 @@ public long count() {
     @Override
     public void refresh() {}
 
+    @Override
+    public ImmutableCacheStatsHolder stats() {
+        // To allow testing of statsTrackingEnabled logic in TSC, return a dummy ImmutableCacheStatsHolder with the
+        // right number of entries, unless statsTrackingEnabled is false
+        return statsHolder.getImmutableCacheStatsHolder(null);
+    }
+
+    @Override
+    public ImmutableCacheStatsHolder stats(String[] levels) {
+        return null;
+    }
+
     @Override
     public void close() {
 
@@ -100,10 +130,12 @@ public static class MockDiskCacheFactory implements Factory {
         public static final String NAME = "mockDiskCache";
         final long delay;
         final int maxSize;
+        final boolean statsTrackingEnabled;
 
-        public MockDiskCacheFactory(long delay, int maxSize) {
+        public MockDiskCacheFactory(long delay, int maxSize, boolean statsTrackingEnabled) {
             this.delay = delay;
             this.maxSize = maxSize;
+            this.statsTrackingEnabled = statsTrackingEnabled;
         }
 
         @Override
@@ -114,6 +146,7 @@ public <K, V> ICache<K, V> create(CacheConfig<K, V> config, CacheType cacheType,
                 .setMaxSize(maxSize)
                 .setDeliberateDelay(delay)
                 .setRemovalListener(config.getRemovalListener())
+                .setStatsTrackingEnabled(config.getStatsTrackingEnabled())
                 .build();
         }
 
@@ -132,7 +165,7 @@ public static class Builder<K, V> extends ICacheBuilder<K, V> {
 
         @Override
         public ICache<K, V> build() {
-            return new MockDiskCache<K, V>(this.maxSize, this.delay, this.getRemovalListener());
+            return new MockDiskCache<K, V>(this.maxSize, this.delay, this.getRemovalListener(), getStatsTrackingEnabled());
         }
 
         public Builder<K, V> setMaxSize(int maxSize) {
@@ -156,4 +189,48 @@ public Builder<K, V> setValueSerializer(Serializer<V, byte[]> valueSerializer) {
         }
 
     }
+
+    /**
+     * Provides a iterator over keys.
+     * @param <K> Type of key
+     * @param <V> Type of value
+     */
+    static class CacheKeyIterator<K, V> implements Iterator<K> {
+        private final Iterator<Map.Entry<K, V>> entryIterator;
+        private final Map<K, V> cache;
+        private final RemovalListener<K, V> removalListener;
+        private K currentKey;
+
+        public CacheKeyIterator(Map<K, V> cache, RemovalListener<K, V> removalListener) {
+            this.entryIterator = cache.entrySet().iterator();
+            this.removalListener = removalListener;
+            this.cache = cache;
+        }
+
+        @Override
+        public boolean hasNext() {
+            return entryIterator.hasNext();
+        }
+
+        @Override
+        public K next() {
+            if (!hasNext()) {
+                throw new NoSuchElementException();
+            }
+            Map.Entry<K, V> entry = entryIterator.next();
+            currentKey = entry.getKey();
+            return currentKey;
+        }
+
+        @Override
+        public void remove() {
+            if (currentKey == null) {
+                throw new IllegalStateException("No element to remove");
+            }
+            V value = cache.get(currentKey);
+            cache.remove(currentKey);
+            this.removalListener.onRemoval(new RemovalNotification<>(currentKey, value, RemovalReason.INVALIDATED));
+            currentKey = null;
+        }
+    }
 }
diff --git a/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/TieredSpilloverCachePluginTests.java b/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/TieredSpilloverCachePluginTests.java
index 1172a48e97c6a..4a96ffe2069ec 100644
--- a/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/TieredSpilloverCachePluginTests.java
+++ b/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/TieredSpilloverCachePluginTests.java
@@ -9,6 +9,8 @@
 package org.opensearch.cache.common.tier;
 
 import org.opensearch.common.cache.ICache;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.test.OpenSearchTestCase;
 
 import java.util.Map;
@@ -16,9 +18,16 @@
 public class TieredSpilloverCachePluginTests extends OpenSearchTestCase {
 
     public void testGetCacheFactoryMap() {
-        TieredSpilloverCachePlugin tieredSpilloverCachePlugin = new TieredSpilloverCachePlugin();
+        TieredSpilloverCachePlugin tieredSpilloverCachePlugin = new TieredSpilloverCachePlugin(Settings.EMPTY);
         Map<String, ICache.Factory> map = tieredSpilloverCachePlugin.getCacheFactoryMap();
         assertNotNull(map.get(TieredSpilloverCache.TieredSpilloverCacheFactory.TIERED_SPILLOVER_CACHE_NAME));
         assertEquals(TieredSpilloverCachePlugin.TIERED_CACHE_SPILLOVER_PLUGIN_NAME, tieredSpilloverCachePlugin.getName());
     }
+
+    public void testGetSettingsWithFeatureFlagOn() {
+        TieredSpilloverCachePlugin tieredSpilloverCachePlugin = new TieredSpilloverCachePlugin(
+            Settings.builder().put(FeatureFlags.PLUGGABLE_CACHE_SETTING.getKey(), true).build()
+        );
+        assertFalse(tieredSpilloverCachePlugin.getSettings().isEmpty());
+    }
 }
diff --git a/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/TieredSpilloverCacheTests.java b/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/TieredSpilloverCacheTests.java
index b132952834f06..6d5ee91326338 100644
--- a/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/TieredSpilloverCacheTests.java
+++ b/modules/cache-common/src/test/java/org/opensearch/cache/common/tier/TieredSpilloverCacheTests.java
@@ -8,27 +8,36 @@
 
 package org.opensearch.cache.common.tier;
 
+import org.opensearch.common.Randomness;
 import org.opensearch.common.cache.CacheType;
 import org.opensearch.common.cache.ICache;
+import org.opensearch.common.cache.ICacheKey;
 import org.opensearch.common.cache.LoadAwareCacheLoader;
 import org.opensearch.common.cache.RemovalListener;
 import org.opensearch.common.cache.RemovalNotification;
 import org.opensearch.common.cache.policy.CachedQueryResult;
 import org.opensearch.common.cache.settings.CacheSettings;
+import org.opensearch.common.cache.stats.ImmutableCacheStats;
+import org.opensearch.common.cache.stats.ImmutableCacheStatsHolder;
 import org.opensearch.common.cache.store.OpenSearchOnHeapCache;
 import org.opensearch.common.cache.store.config.CacheConfig;
 import org.opensearch.common.cache.store.settings.OpenSearchOnHeapCacheSettings;
 import org.opensearch.common.metrics.CounterMetric;
+import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.test.OpenSearchTestCase;
+import org.junit.Before;
 
+import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Random;
 import java.util.UUID;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.CountDownLatch;
@@ -38,16 +47,32 @@
 import java.util.function.Function;
 import java.util.function.Predicate;
 
+import static org.opensearch.cache.common.tier.TieredSpilloverCacheSettings.DISK_CACHE_ENABLED_SETTING_MAP;
+import static org.opensearch.cache.common.tier.TieredSpilloverCacheSettings.TOOK_TIME_POLICY_CONCRETE_SETTINGS_MAP;
+import static org.opensearch.cache.common.tier.TieredSpilloverCacheStatsHolder.TIER_DIMENSION_NAME;
+import static org.opensearch.cache.common.tier.TieredSpilloverCacheStatsHolder.TIER_DIMENSION_VALUE_DISK;
+import static org.opensearch.cache.common.tier.TieredSpilloverCacheStatsHolder.TIER_DIMENSION_VALUE_ON_HEAP;
 import static org.opensearch.common.cache.store.settings.OpenSearchOnHeapCacheSettings.MAXIMUM_SIZE_IN_BYTES_KEY;
 
 public class TieredSpilloverCacheTests extends OpenSearchTestCase {
+    static final List<String> dimensionNames = List.of("dim1", "dim2", "dim3");
+
+    private ClusterSettings clusterSettings;
+
+    @Before
+    public void setup() {
+        Settings settings = Settings.EMPTY;
+        clusterSettings = new ClusterSettings(settings, new HashSet<>());
+        clusterSettings.registerSetting(TOOK_TIME_POLICY_CONCRETE_SETTINGS_MAP.get(CacheType.INDICES_REQUEST_CACHE));
+        clusterSettings.registerSetting(DISK_CACHE_ENABLED_SETTING_MAP.get(CacheType.INDICES_REQUEST_CACHE));
+    }
 
     public void testComputeIfAbsentWithoutAnyOnHeapCacheEviction() throws Exception {
         int onHeapCacheSize = randomIntBetween(10, 30);
         int keyValueSize = 50;
 
         MockCacheRemovalListener<String, String> removalListener = new MockCacheRemovalListener<>();
-        TieredSpilloverCache<String, String> tieredSpilloverCache = intializeTieredSpilloverCache(
+        TieredSpilloverCache<String, String> tieredSpilloverCache = initializeTieredSpilloverCache(
             keyValueSize,
             randomIntBetween(1, 4),
             removalListener,
@@ -62,15 +87,18 @@ public void testComputeIfAbsentWithoutAnyOnHeapCacheEviction() throws Exception
             0
         );
         int numOfItems1 = randomIntBetween(1, onHeapCacheSize / 2 - 1);
-        List<String> keys = new ArrayList<>();
+        List<ICacheKey<String>> keys = new ArrayList<>();
         // Put values in cache.
         for (int iter = 0; iter < numOfItems1; iter++) {
-            String key = UUID.randomUUID().toString();
+            ICacheKey<String> key = getICacheKey(UUID.randomUUID().toString());
             keys.add(key);
-            LoadAwareCacheLoader<String, String> tieredCacheLoader = getLoadAwareCacheLoader();
+            LoadAwareCacheLoader<ICacheKey<String>, String> tieredCacheLoader = getLoadAwareCacheLoader();
             tieredSpilloverCache.computeIfAbsent(key, tieredCacheLoader);
         }
         assertEquals(0, removalListener.evictionsMetric.count());
+        assertEquals(numOfItems1, getMissesForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+        assertEquals(0, getHitsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+        assertEquals(0, getEvictionsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
 
         // Try to hit cache again with some randomization.
         int numOfItems2 = randomIntBetween(1, onHeapCacheSize / 2 - 1);
@@ -84,11 +112,18 @@ public void testComputeIfAbsentWithoutAnyOnHeapCacheEviction() throws Exception
                 tieredSpilloverCache.computeIfAbsent(keys.get(index), getLoadAwareCacheLoader());
             } else {
                 // Hit cache with randomized key which is expected to miss cache always.
-                tieredSpilloverCache.computeIfAbsent(UUID.randomUUID().toString(), getLoadAwareCacheLoader());
+                tieredSpilloverCache.computeIfAbsent(getICacheKey(UUID.randomUUID().toString()), getLoadAwareCacheLoader());
                 cacheMiss++;
             }
         }
         assertEquals(0, removalListener.evictionsMetric.count());
+        assertEquals(cacheHit, getHitsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+        assertEquals(numOfItems1 + cacheMiss, getMissesForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+        assertEquals(0, getEvictionsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+
+        assertEquals(0, getHitsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK));
+        assertEquals(numOfItems1 + cacheMiss, getMissesForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK));
+        assertEquals(0, getEvictionsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK));
     }
 
     public void testComputeIfAbsentWithFactoryBasedCacheCreation() throws Exception {
@@ -132,30 +167,49 @@ public void testComputeIfAbsentWithFactoryBasedCacheCreation() throws Exception
                 .setWeigher((k, v) -> keyValueSize)
                 .setRemovalListener(removalListener)
                 .setSettings(settings)
+                .setDimensionNames(dimensionNames)
                 .setCachedResultParser(s -> new CachedQueryResult.PolicyValues(20_000_000L)) // Values will always appear to have taken
                                                                                              // 20_000_000 ns = 20 ms to compute
+                .setClusterSettings(clusterSettings)
                 .build(),
             CacheType.INDICES_REQUEST_CACHE,
             Map.of(
                 OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory.NAME,
                 new OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory(),
                 MockDiskCache.MockDiskCacheFactory.NAME,
-                new MockDiskCache.MockDiskCacheFactory(0, randomIntBetween(100, 300))
+                new MockDiskCache.MockDiskCacheFactory(0, randomIntBetween(100, 300), false)
             )
         );
 
         TieredSpilloverCache<String, String> tieredSpilloverCache = (TieredSpilloverCache<String, String>) tieredSpilloverICache;
 
         int numOfItems1 = randomIntBetween(onHeapCacheSize + 1, totalSize);
+        List<ICacheKey<String>> onHeapKeys = new ArrayList<>();
+        List<ICacheKey<String>> diskTierKeys = new ArrayList<>();
         for (int iter = 0; iter < numOfItems1; iter++) {
             String key = UUID.randomUUID().toString();
-            LoadAwareCacheLoader<String, String> tieredCacheLoader = getLoadAwareCacheLoader();
-            tieredSpilloverCache.computeIfAbsent(key, tieredCacheLoader);
+            LoadAwareCacheLoader<ICacheKey<String>, String> tieredCacheLoader = getLoadAwareCacheLoader();
+            tieredSpilloverCache.computeIfAbsent(getICacheKey(key), tieredCacheLoader);
         }
-        // Verify on heap cache size.
+
+        int expectedDiskEntries = numOfItems1 - onHeapCacheSize;
+        tieredSpilloverCache.getOnHeapCache().keys().forEach(onHeapKeys::add);
+        tieredSpilloverCache.getDiskCache().keys().forEach(diskTierKeys::add);
+        // Verify on heap cache stats.
         assertEquals(onHeapCacheSize, tieredSpilloverCache.getOnHeapCache().count());
-        // Verify disk cache size.
-        assertEquals(numOfItems1 - onHeapCacheSize, tieredSpilloverCache.getDiskCache().count());
+        assertEquals(onHeapCacheSize, getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+        assertEquals(numOfItems1, getMissesForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+        assertEquals(0, getHitsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+        assertEquals(expectedDiskEntries, getEvictionsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+        assertEquals(onHeapCacheSize * keyValueSize, getSizeInBytesForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+
+        // Verify disk cache stats.
+        assertEquals(expectedDiskEntries, tieredSpilloverCache.getDiskCache().count());
+        assertEquals(expectedDiskEntries, getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK));
+        assertEquals(0, getHitsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK));
+        assertEquals(numOfItems1, getMissesForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK));
+        assertEquals(0, getEvictionsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK));
+        assertEquals(expectedDiskEntries * keyValueSize, getSizeInBytesForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK));
     }
 
     public void testWithFactoryCreationWithOnHeapCacheNotPresent() {
@@ -198,7 +252,7 @@ public void testWithFactoryCreationWithOnHeapCacheNotPresent() {
                     OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory.NAME,
                     new OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory(),
                     MockDiskCache.MockDiskCacheFactory.NAME,
-                    new MockDiskCache.MockDiskCacheFactory(0, randomIntBetween(100, 300))
+                    new MockDiskCache.MockDiskCacheFactory(0, randomIntBetween(100, 300), false)
                 )
             )
         );
@@ -243,7 +297,7 @@ public void testWithFactoryCreationWithDiskCacheNotPresent() {
                     OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory.NAME,
                     new OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory(),
                     MockDiskCache.MockDiskCacheFactory.NAME,
-                    new MockDiskCache.MockDiskCacheFactory(0, randomIntBetween(100, 300))
+                    new MockDiskCache.MockDiskCacheFactory(0, randomIntBetween(100, 300), false)
                 )
             )
         );
@@ -264,6 +318,7 @@ public void testComputeIfAbsentWithEvictionsFromOnHeapCache() throws Exception {
             .setKeyType(String.class)
             .setWeigher((k, v) -> keyValueSize)
             .setRemovalListener(removalListener)
+            .setDimensionNames(dimensionNames)
             .setSettings(
                 Settings.builder()
                     .put(
@@ -279,9 +334,10 @@ public void testComputeIfAbsentWithEvictionsFromOnHeapCache() throws Exception {
                     )
                     .build()
             )
+            .setClusterSettings(clusterSettings)
             .build();
 
-        ICache.Factory mockDiskCacheFactory = new MockDiskCache.MockDiskCacheFactory(0, diskCacheSize);
+        ICache.Factory mockDiskCacheFactory = new MockDiskCache.MockDiskCacheFactory(0, diskCacheSize, false);
 
         TieredSpilloverCache<String, String> tieredSpilloverCache = new TieredSpilloverCache.Builder<String, String>()
             .setOnHeapCacheFactory(onHeapCacheFactory)
@@ -293,20 +349,26 @@ public void testComputeIfAbsentWithEvictionsFromOnHeapCache() throws Exception {
 
         // Put values in cache more than it's size and cause evictions from onHeap.
         int numOfItems1 = randomIntBetween(onHeapCacheSize + 1, totalSize);
-        List<String> onHeapKeys = new ArrayList<>();
-        List<String> diskTierKeys = new ArrayList<>();
+        List<ICacheKey<String>> onHeapKeys = new ArrayList<>();
+        List<ICacheKey<String>> diskTierKeys = new ArrayList<>();
         for (int iter = 0; iter < numOfItems1; iter++) {
-            String key = UUID.randomUUID().toString();
-            LoadAwareCacheLoader<String, String> tieredCacheLoader = getLoadAwareCacheLoader();
+            ICacheKey<String> key = getICacheKey(UUID.randomUUID().toString());
+            LoadAwareCacheLoader<ICacheKey<String>, String> tieredCacheLoader = getLoadAwareCacheLoader();
             tieredSpilloverCache.computeIfAbsent(key, tieredCacheLoader);
         }
 
+        long actualDiskCacheSize = tieredSpilloverCache.getDiskCache().count();
+
+        assertEquals(numOfItems1, getMissesForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+        assertEquals(0, getHitsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+        assertEquals(actualDiskCacheSize, getEvictionsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+        assertEquals(onHeapCacheSize, getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+        assertEquals(onHeapCacheSize * keyValueSize, getSizeInBytesForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+        assertEquals(actualDiskCacheSize, getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK));
+
         tieredSpilloverCache.getOnHeapCache().keys().forEach(onHeapKeys::add);
         tieredSpilloverCache.getDiskCache().keys().forEach(diskTierKeys::add);
 
-        assertEquals(tieredSpilloverCache.getOnHeapCache().count(), onHeapKeys.size());
-        assertEquals(tieredSpilloverCache.getDiskCache().count(), diskTierKeys.size());
-
         // Try to hit cache again with some randomization.
         int numOfItems2 = randomIntBetween(50, 200);
         int onHeapCacheHit = 0;
@@ -316,23 +378,29 @@ public void testComputeIfAbsentWithEvictionsFromOnHeapCache() throws Exception {
             if (randomBoolean()) { // Hit cache with key stored in onHeap cache.
                 onHeapCacheHit++;
                 int index = randomIntBetween(0, onHeapKeys.size() - 1);
-                LoadAwareCacheLoader<String, String> loadAwareCacheLoader = getLoadAwareCacheLoader();
+                LoadAwareCacheLoader<ICacheKey<String>, String> loadAwareCacheLoader = getLoadAwareCacheLoader();
                 tieredSpilloverCache.computeIfAbsent(onHeapKeys.get(index), loadAwareCacheLoader);
                 assertFalse(loadAwareCacheLoader.isLoaded());
             } else { // Hit cache with key stored in disk cache.
                 diskCacheHit++;
                 int index = randomIntBetween(0, diskTierKeys.size() - 1);
-                LoadAwareCacheLoader<String, String> loadAwareCacheLoader = getLoadAwareCacheLoader();
+                LoadAwareCacheLoader<ICacheKey<String>, String> loadAwareCacheLoader = getLoadAwareCacheLoader();
                 tieredSpilloverCache.computeIfAbsent(diskTierKeys.get(index), loadAwareCacheLoader);
                 assertFalse(loadAwareCacheLoader.isLoaded());
             }
         }
-        for (int iter = 0; iter < randomIntBetween(50, 200); iter++) {
+        int numRandom = randomIntBetween(50, 200);
+        for (int iter = 0; iter < numRandom; iter++) {
             // Hit cache with randomized key which is expected to miss cache always.
-            LoadAwareCacheLoader<String, String> tieredCacheLoader = getLoadAwareCacheLoader();
-            tieredSpilloverCache.computeIfAbsent(UUID.randomUUID().toString(), tieredCacheLoader);
+            LoadAwareCacheLoader<ICacheKey<String>, String> tieredCacheLoader = getLoadAwareCacheLoader();
+            tieredSpilloverCache.computeIfAbsent(getICacheKey(UUID.randomUUID().toString()), tieredCacheLoader);
             cacheMiss++;
         }
+
+        assertEquals(numOfItems1 + cacheMiss + diskCacheHit, getMissesForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+        assertEquals(onHeapCacheHit, getHitsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+        assertEquals(cacheMiss + numOfItems1, getMissesForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK));
+        assertEquals(diskCacheHit, getHitsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK));
     }
 
     public void testComputeIfAbsentWithEvictionsFromTieredCache() throws Exception {
@@ -342,7 +410,7 @@ public void testComputeIfAbsentWithEvictionsFromTieredCache() throws Exception {
         int keyValueSize = 50;
 
         MockCacheRemovalListener<String, String> removalListener = new MockCacheRemovalListener<>();
-        TieredSpilloverCache<String, String> tieredSpilloverCache = intializeTieredSpilloverCache(
+        TieredSpilloverCache<String, String> tieredSpilloverCache = initializeTieredSpilloverCache(
             keyValueSize,
             diskCacheSize,
             removalListener,
@@ -358,11 +426,17 @@ public void testComputeIfAbsentWithEvictionsFromTieredCache() throws Exception {
         );
         int numOfItems = randomIntBetween(totalSize + 1, totalSize * 3);
         for (int iter = 0; iter < numOfItems; iter++) {
-            LoadAwareCacheLoader<String, String> tieredCacheLoader = getLoadAwareCacheLoader();
-            tieredSpilloverCache.computeIfAbsent(UUID.randomUUID().toString(), tieredCacheLoader);
+            LoadAwareCacheLoader<ICacheKey<String>, String> tieredCacheLoader = getLoadAwareCacheLoader();
+            tieredSpilloverCache.computeIfAbsent(getICacheKey(UUID.randomUUID().toString()), tieredCacheLoader);
         }
-        int evictions = numOfItems - (totalSize);
+
+        int evictions = numOfItems - (totalSize); // Evictions from the cache as a whole
         assertEquals(evictions, removalListener.evictionsMetric.count());
+        assertEquals(evictions, getEvictionsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK));
+        assertEquals(
+            evictions + getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK),
+            getEvictionsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP)
+        );
     }
 
     public void testGetAndCount() throws Exception {
@@ -372,7 +446,7 @@ public void testGetAndCount() throws Exception {
         int totalSize = onHeapCacheSize + diskCacheSize;
 
         MockCacheRemovalListener<String, String> removalListener = new MockCacheRemovalListener<>();
-        TieredSpilloverCache<String, String> tieredSpilloverCache = intializeTieredSpilloverCache(
+        TieredSpilloverCache<String, String> tieredSpilloverCache = initializeTieredSpilloverCache(
             keyValueSize,
             diskCacheSize,
             removalListener,
@@ -388,17 +462,17 @@ public void testGetAndCount() throws Exception {
         );
 
         int numOfItems1 = randomIntBetween(onHeapCacheSize + 1, totalSize);
-        List<String> onHeapKeys = new ArrayList<>();
-        List<String> diskTierKeys = new ArrayList<>();
+        List<ICacheKey<String>> onHeapKeys = new ArrayList<>();
+        List<ICacheKey<String>> diskTierKeys = new ArrayList<>();
         for (int iter = 0; iter < numOfItems1; iter++) {
-            String key = UUID.randomUUID().toString();
+            ICacheKey<String> key = getICacheKey(UUID.randomUUID().toString());
             if (iter > (onHeapCacheSize - 1)) {
                 // All these are bound to go to disk based cache.
                 diskTierKeys.add(key);
             } else {
                 onHeapKeys.add(key);
             }
-            LoadAwareCacheLoader<String, String> loadAwareCacheLoader = getLoadAwareCacheLoader();
+            LoadAwareCacheLoader<ICacheKey<String>, String> loadAwareCacheLoader = getLoadAwareCacheLoader();
             tieredSpilloverCache.computeIfAbsent(key, loadAwareCacheLoader);
         }
 
@@ -412,19 +486,19 @@ public void testGetAndCount() throws Exception {
                     assertNotNull(tieredSpilloverCache.get(diskTierKeys.get(index)));
                 }
             } else {
-                assertNull(tieredSpilloverCache.get(UUID.randomUUID().toString()));
+                assertNull(tieredSpilloverCache.get(getICacheKey(UUID.randomUUID().toString())));
             }
         }
         assertEquals(numOfItems1, tieredSpilloverCache.count());
     }
 
-    public void testPut() {
+    public void testPut() throws Exception {
         int onHeapCacheSize = randomIntBetween(10, 30);
         int diskCacheSize = randomIntBetween(onHeapCacheSize + 1, 100);
         int keyValueSize = 50;
 
         MockCacheRemovalListener<String, String> removalListener = new MockCacheRemovalListener<>();
-        TieredSpilloverCache<String, String> tieredSpilloverCache = intializeTieredSpilloverCache(
+        TieredSpilloverCache<String, String> tieredSpilloverCache = initializeTieredSpilloverCache(
             keyValueSize,
             diskCacheSize,
             removalListener,
@@ -438,9 +512,10 @@ public void testPut() {
                 .build(),
             0
         );
-        String key = UUID.randomUUID().toString();
+        ICacheKey<String> key = getICacheKey(UUID.randomUUID().toString());
         String value = UUID.randomUUID().toString();
         tieredSpilloverCache.put(key, value);
+        assertEquals(1, getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
         assertEquals(1, tieredSpilloverCache.count());
     }
 
@@ -451,7 +526,7 @@ public void testPutAndVerifyNewItemsArePresentOnHeapCache() throws Exception {
 
         MockCacheRemovalListener<String, String> removalListener = new MockCacheRemovalListener<>();
 
-        TieredSpilloverCache<String, String> tieredSpilloverCache = intializeTieredSpilloverCache(
+        TieredSpilloverCache<String, String> tieredSpilloverCache = initializeTieredSpilloverCache(
             keyValueSize,
             diskCacheSize,
             removalListener,
@@ -471,61 +546,41 @@ public void testPutAndVerifyNewItemsArePresentOnHeapCache() throws Exception {
         );
 
         for (int i = 0; i < onHeapCacheSize; i++) {
-            tieredSpilloverCache.computeIfAbsent(UUID.randomUUID().toString(), new LoadAwareCacheLoader<>() {
-                @Override
-                public boolean isLoaded() {
-                    return false;
-                }
-
-                @Override
-                public String load(String key) {
-                    return UUID.randomUUID().toString();
-                }
-            });
+            tieredSpilloverCache.computeIfAbsent(getICacheKey(UUID.randomUUID().toString()), getLoadAwareCacheLoader());
         }
 
-        assertEquals(onHeapCacheSize, tieredSpilloverCache.getOnHeapCache().count());
-        assertEquals(0, tieredSpilloverCache.getDiskCache().count());
+        assertEquals(onHeapCacheSize, getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+        assertEquals(0, getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK));
 
         // Again try to put OnHeap cache capacity amount of new items.
-        List<String> newKeyList = new ArrayList<>();
+        List<ICacheKey<String>> newKeyList = new ArrayList<>();
         for (int i = 0; i < onHeapCacheSize; i++) {
-            newKeyList.add(UUID.randomUUID().toString());
+            newKeyList.add(getICacheKey(UUID.randomUUID().toString()));
         }
 
         for (int i = 0; i < newKeyList.size(); i++) {
-            tieredSpilloverCache.computeIfAbsent(newKeyList.get(i), new LoadAwareCacheLoader<>() {
-                @Override
-                public boolean isLoaded() {
-                    return false;
-                }
-
-                @Override
-                public String load(String key) {
-                    return UUID.randomUUID().toString();
-                }
-            });
+            tieredSpilloverCache.computeIfAbsent(newKeyList.get(i), getLoadAwareCacheLoader());
         }
 
         // Verify that new items are part of onHeap cache.
-        List<String> actualOnHeapCacheKeys = new ArrayList<>();
+        List<ICacheKey<String>> actualOnHeapCacheKeys = new ArrayList<>();
         tieredSpilloverCache.getOnHeapCache().keys().forEach(actualOnHeapCacheKeys::add);
 
         assertEquals(newKeyList.size(), actualOnHeapCacheKeys.size());
         for (int i = 0; i < actualOnHeapCacheKeys.size(); i++) {
             assertTrue(newKeyList.contains(actualOnHeapCacheKeys.get(i)));
         }
-        assertEquals(onHeapCacheSize, tieredSpilloverCache.getOnHeapCache().count());
-        assertEquals(onHeapCacheSize, tieredSpilloverCache.getDiskCache().count());
+        assertEquals(onHeapCacheSize, getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+        assertEquals(onHeapCacheSize, getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK));
     }
 
-    public void testInvalidate() {
+    public void testInvalidate() throws Exception {
         int onHeapCacheSize = 1;
         int diskCacheSize = 10;
         int keyValueSize = 20;
 
         MockCacheRemovalListener<String, String> removalListener = new MockCacheRemovalListener<>();
-        TieredSpilloverCache<String, String> tieredSpilloverCache = intializeTieredSpilloverCache(
+        TieredSpilloverCache<String, String> tieredSpilloverCache = initializeTieredSpilloverCache(
             keyValueSize,
             diskCacheSize,
             removalListener,
@@ -539,23 +594,33 @@ public void testInvalidate() {
                 .build(),
             0
         );
-        String key = UUID.randomUUID().toString();
+        ICacheKey<String> key = getICacheKey(UUID.randomUUID().toString());
         String value = UUID.randomUUID().toString();
         // First try to invalidate without the key present in cache.
         tieredSpilloverCache.invalidate(key);
+        assertEquals(0, getEvictionsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
 
         // Now try to invalidate with the key present in onHeap cache.
         tieredSpilloverCache.put(key, value);
         tieredSpilloverCache.invalidate(key);
+        assertEquals(0, getEvictionsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+        // Evictions metric shouldn't increase for invalidations.
         assertEquals(0, tieredSpilloverCache.count());
 
         tieredSpilloverCache.put(key, value);
         // Put another key/value so that one of the item is evicted to disk cache.
-        String key2 = UUID.randomUUID().toString();
+        ICacheKey<String> key2 = getICacheKey(UUID.randomUUID().toString());
         tieredSpilloverCache.put(key2, UUID.randomUUID().toString());
+
         assertEquals(2, tieredSpilloverCache.count());
-        // Again invalidate older key
+        assertEquals(1, getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+        assertEquals(1, getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK));
+
+        // Again invalidate older key, leaving one in heap tier and zero in disk tier
         tieredSpilloverCache.invalidate(key);
+        assertEquals(0, getEvictionsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK));
+        assertEquals(0, getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK));
+        assertEquals(1, getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
         assertEquals(1, tieredSpilloverCache.count());
     }
 
@@ -565,7 +630,7 @@ public void testCacheKeys() throws Exception {
         int keyValueSize = 50;
 
         MockCacheRemovalListener<String, String> removalListener = new MockCacheRemovalListener<>();
-        TieredSpilloverCache<String, String> tieredSpilloverCache = intializeTieredSpilloverCache(
+        TieredSpilloverCache<String, String> tieredSpilloverCache = initializeTieredSpilloverCache(
             keyValueSize,
             diskCacheSize,
             removalListener,
@@ -579,46 +644,46 @@ public void testCacheKeys() throws Exception {
                 .build(),
             0
         );
-        List<String> onHeapKeys = new ArrayList<>();
-        List<String> diskTierKeys = new ArrayList<>();
+        List<ICacheKey<String>> onHeapKeys = new ArrayList<>();
+        List<ICacheKey<String>> diskTierKeys = new ArrayList<>();
         // During first round add onHeapCacheSize entries. Will go to onHeap cache initially.
         for (int i = 0; i < onHeapCacheSize; i++) {
-            String key = UUID.randomUUID().toString();
+            ICacheKey<String> key = getICacheKey(UUID.randomUUID().toString());
             diskTierKeys.add(key);
             tieredSpilloverCache.computeIfAbsent(key, getLoadAwareCacheLoader());
         }
         // In another round, add another onHeapCacheSize entries. These will go to onHeap and above ones will be
         // evicted to onDisk cache.
         for (int i = 0; i < onHeapCacheSize; i++) {
-            String key = UUID.randomUUID().toString();
+            ICacheKey<String> key = getICacheKey(UUID.randomUUID().toString());
             onHeapKeys.add(key);
             tieredSpilloverCache.computeIfAbsent(key, getLoadAwareCacheLoader());
         }
 
-        List<String> actualOnHeapKeys = new ArrayList<>();
-        List<String> actualOnDiskKeys = new ArrayList<>();
-        Iterable<String> onHeapiterable = tieredSpilloverCache.getOnHeapCache().keys();
-        Iterable<String> onDiskiterable = tieredSpilloverCache.getDiskCache().keys();
+        List<ICacheKey<String>> actualOnHeapKeys = new ArrayList<>();
+        List<ICacheKey<String>> actualOnDiskKeys = new ArrayList<>();
+        Iterable<ICacheKey<String>> onHeapiterable = tieredSpilloverCache.getOnHeapCache().keys();
+        Iterable<ICacheKey<String>> onDiskiterable = tieredSpilloverCache.getDiskCache().keys();
         onHeapiterable.iterator().forEachRemaining(actualOnHeapKeys::add);
         onDiskiterable.iterator().forEachRemaining(actualOnDiskKeys::add);
-        for (String onHeapKey : onHeapKeys) {
+        for (ICacheKey<String> onHeapKey : onHeapKeys) {
             assertTrue(actualOnHeapKeys.contains(onHeapKey));
         }
-        for (String onDiskKey : actualOnDiskKeys) {
+        for (ICacheKey<String> onDiskKey : actualOnDiskKeys) {
             assertTrue(actualOnDiskKeys.contains(onDiskKey));
         }
 
         // Testing keys() which returns all keys.
-        List<String> actualMergedKeys = new ArrayList<>();
-        List<String> expectedMergedKeys = new ArrayList<>();
+        List<ICacheKey<String>> actualMergedKeys = new ArrayList<>();
+        List<ICacheKey<String>> expectedMergedKeys = new ArrayList<>();
         expectedMergedKeys.addAll(onHeapKeys);
         expectedMergedKeys.addAll(diskTierKeys);
 
-        Iterable<String> mergedIterable = tieredSpilloverCache.keys();
+        Iterable<ICacheKey<String>> mergedIterable = tieredSpilloverCache.keys();
         mergedIterable.iterator().forEachRemaining(actualMergedKeys::add);
 
         assertEquals(expectedMergedKeys.size(), actualMergedKeys.size());
-        for (String key : expectedMergedKeys) {
+        for (ICacheKey<String> key : expectedMergedKeys) {
             assertTrue(actualMergedKeys.contains(key));
         }
     }
@@ -627,7 +692,7 @@ public void testRefresh() {
         int diskCacheSize = randomIntBetween(60, 100);
 
         MockCacheRemovalListener<String, String> removalListener = new MockCacheRemovalListener<>();
-        TieredSpilloverCache<String, String> tieredSpilloverCache = intializeTieredSpilloverCache(
+        TieredSpilloverCache<String, String> tieredSpilloverCache = initializeTieredSpilloverCache(
             50,
             diskCacheSize,
             removalListener,
@@ -644,7 +709,7 @@ public void testInvalidateAll() throws Exception {
         int totalSize = onHeapCacheSize + diskCacheSize;
 
         MockCacheRemovalListener<String, String> removalListener = new MockCacheRemovalListener<>();
-        TieredSpilloverCache<String, String> tieredSpilloverCache = intializeTieredSpilloverCache(
+        TieredSpilloverCache<String, String> tieredSpilloverCache = initializeTieredSpilloverCache(
             keyValueSize,
             diskCacheSize,
             removalListener,
@@ -660,17 +725,17 @@ public void testInvalidateAll() throws Exception {
         );
         // Put values in cache more than it's size and cause evictions from onHeap.
         int numOfItems1 = randomIntBetween(onHeapCacheSize + 1, totalSize);
-        List<String> onHeapKeys = new ArrayList<>();
-        List<String> diskTierKeys = new ArrayList<>();
+        List<ICacheKey<String>> onHeapKeys = new ArrayList<>();
+        List<ICacheKey<String>> diskTierKeys = new ArrayList<>();
         for (int iter = 0; iter < numOfItems1; iter++) {
-            String key = UUID.randomUUID().toString();
+            ICacheKey<String> key = getICacheKey(UUID.randomUUID().toString());
             if (iter > (onHeapCacheSize - 1)) {
                 // All these are bound to go to disk based cache.
                 diskTierKeys.add(key);
             } else {
                 onHeapKeys.add(key);
             }
-            LoadAwareCacheLoader<String, String> tieredCacheLoader = getLoadAwareCacheLoader();
+            LoadAwareCacheLoader<ICacheKey<String>, String> tieredCacheLoader = getLoadAwareCacheLoader();
             tieredSpilloverCache.computeIfAbsent(key, tieredCacheLoader);
         }
         assertEquals(numOfItems1, tieredSpilloverCache.count());
@@ -693,7 +758,7 @@ public void testComputeIfAbsentConcurrently() throws Exception {
             )
             .build();
 
-        TieredSpilloverCache<String, String> tieredSpilloverCache = intializeTieredSpilloverCache(
+        TieredSpilloverCache<String, String> tieredSpilloverCache = initializeTieredSpilloverCache(
             keyValueSize,
             diskCacheSize,
             removalListener,
@@ -702,19 +767,19 @@ public void testComputeIfAbsentConcurrently() throws Exception {
         );
 
         int numberOfSameKeys = randomIntBetween(10, onHeapCacheSize - 1);
-        String key = UUID.randomUUID().toString();
+        ICacheKey<String> key = getICacheKey(UUID.randomUUID().toString());
         String value = UUID.randomUUID().toString();
 
         Thread[] threads = new Thread[numberOfSameKeys];
         Phaser phaser = new Phaser(numberOfSameKeys + 1);
         CountDownLatch countDownLatch = new CountDownLatch(numberOfSameKeys); // To wait for all threads to finish.
 
-        List<LoadAwareCacheLoader<String, String>> loadAwareCacheLoaderList = new CopyOnWriteArrayList<>();
+        List<LoadAwareCacheLoader<ICacheKey<String>, String>> loadAwareCacheLoaderList = new CopyOnWriteArrayList<>();
 
         for (int i = 0; i < numberOfSameKeys; i++) {
             threads[i] = new Thread(() -> {
                 try {
-                    LoadAwareCacheLoader<String, String> loadAwareCacheLoader = new LoadAwareCacheLoader<>() {
+                    LoadAwareCacheLoader<ICacheKey<String>, String> loadAwareCacheLoader = new LoadAwareCacheLoader<>() {
                         boolean isLoaded = false;
 
                         @Override
@@ -723,7 +788,7 @@ public boolean isLoaded() {
                         }
 
                         @Override
-                        public String load(String key) {
+                        public String load(ICacheKey<String> key) {
                             isLoaded = true;
                             return value;
                         }
@@ -743,12 +808,15 @@ public String load(String key) {
         int numberOfTimesKeyLoaded = 0;
         assertEquals(numberOfSameKeys, loadAwareCacheLoaderList.size());
         for (int i = 0; i < loadAwareCacheLoaderList.size(); i++) {
-            LoadAwareCacheLoader<String, String> loader = loadAwareCacheLoaderList.get(i);
+            LoadAwareCacheLoader<ICacheKey<String>, String> loader = loadAwareCacheLoaderList.get(i);
             if (loader.isLoaded()) {
                 numberOfTimesKeyLoaded++;
             }
         }
         assertEquals(1, numberOfTimesKeyLoaded); // It should be loaded only once.
+        // We should see only one heap miss, and the rest hits
+        assertEquals(1, getMissesForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+        assertEquals(numberOfSameKeys - 1, getHitsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
     }
 
     public void testConcurrencyForEvictionFlowFromOnHeapToDiskTier() throws Exception {
@@ -757,7 +825,7 @@ public void testConcurrencyForEvictionFlowFromOnHeapToDiskTier() throws Exceptio
         MockCacheRemovalListener<String, String> removalListener = new MockCacheRemovalListener<>();
 
         ICache.Factory onHeapCacheFactory = new OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory();
-        ICache.Factory diskCacheFactory = new MockDiskCache.MockDiskCacheFactory(500, diskCacheSize);
+        ICache.Factory diskCacheFactory = new MockDiskCache.MockDiskCacheFactory(500, diskCacheSize, false);
         CacheConfig<String, String> cacheConfig = new CacheConfig.Builder<String, String>().setKeyType(String.class)
             .setKeyType(String.class)
             .setWeigher((k, v) -> 150)
@@ -777,6 +845,8 @@ public void testConcurrencyForEvictionFlowFromOnHeapToDiskTier() throws Exceptio
                     )
                     .build()
             )
+            .setClusterSettings(clusterSettings)
+            .setDimensionNames(dimensionNames)
             .build();
         TieredSpilloverCache<String, String> tieredSpilloverCache = new TieredSpilloverCache.Builder<String, String>()
             .setOnHeapCacheFactory(onHeapCacheFactory)
@@ -786,26 +856,17 @@ public void testConcurrencyForEvictionFlowFromOnHeapToDiskTier() throws Exceptio
             .setCacheType(CacheType.INDICES_REQUEST_CACHE)
             .build();
 
-        String keyToBeEvicted = "key1";
-        String secondKey = "key2";
+        ICacheKey<String> keyToBeEvicted = getICacheKey("key1");
+        ICacheKey<String> secondKey = getICacheKey("key2");
 
         // Put first key on tiered cache. Will go into onHeap cache.
-        tieredSpilloverCache.computeIfAbsent(keyToBeEvicted, new LoadAwareCacheLoader<>() {
-            @Override
-            public boolean isLoaded() {
-                return false;
-            }
-
-            @Override
-            public String load(String key) {
-                return UUID.randomUUID().toString();
-            }
-        });
+        tieredSpilloverCache.computeIfAbsent(keyToBeEvicted, getLoadAwareCacheLoader());
+        assertEquals(1, getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
         CountDownLatch countDownLatch = new CountDownLatch(1);
         CountDownLatch countDownLatch1 = new CountDownLatch(1);
         // Put second key on tiered cache. Will cause eviction of first key from onHeap cache and should go into
         // disk cache.
-        LoadAwareCacheLoader<String, String> loadAwareCacheLoader = getLoadAwareCacheLoader();
+        LoadAwareCacheLoader<ICacheKey<String>, String> loadAwareCacheLoader = getLoadAwareCacheLoader();
         Thread thread = new Thread(() -> {
             try {
                 tieredSpilloverCache.computeIfAbsent(secondKey, loadAwareCacheLoader);
@@ -816,7 +877,7 @@ public String load(String key) {
         });
         thread.start();
         assertBusy(() -> { assertTrue(loadAwareCacheLoader.isLoaded()); }, 100, TimeUnit.MILLISECONDS); // We wait for new key to be loaded
-                                                                                                        // after which it eviction flow is
+        // after which it eviction flow is
         // guaranteed to occur.
         ICache<String, String> onDiskCache = tieredSpilloverCache.getDiskCache();
 
@@ -835,18 +896,14 @@ public String load(String key) {
         countDownLatch.await();
         assertNotNull(actualValue.get());
         countDownLatch1.await();
+
         assertEquals(1, tieredSpilloverCache.getOnHeapCache().count());
         assertEquals(1, onDiskCache.count());
-        assertNotNull(onDiskCache.get(keyToBeEvicted));
-    }
 
-    class MockCacheRemovalListener<K, V> implements RemovalListener<K, V> {
-        final CounterMetric evictionsMetric = new CounterMetric();
-
-        @Override
-        public void onRemoval(RemovalNotification<K, V> notification) {
-            evictionsMetric.inc();
-        }
+        assertEquals(1, getEvictionsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+        assertEquals(1, getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP));
+        assertEquals(1, getItemsForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK));
+        assertNotNull(onDiskCache.get(keyToBeEvicted));
     }
 
     public void testDiskTierPolicies() throws Exception {
@@ -887,26 +944,14 @@ public void testDiskTierPolicies() throws Exception {
         keyValuePairs.put("key5", "");
         expectedOutputs.put("key5", false);
 
-        LoadAwareCacheLoader<String, String> loader = new LoadAwareCacheLoader<String, String>() {
-            boolean isLoaded = false;
-
-            @Override
-            public boolean isLoaded() {
-                return isLoaded;
-            }
-
-            @Override
-            public String load(String key) throws Exception {
-                isLoaded = true;
-                return keyValuePairs.get(key);
-            }
-        };
+        LoadAwareCacheLoader<ICacheKey<String>, String> loader = getLoadAwareCacheLoader(keyValuePairs);
 
         for (String key : keyValuePairs.keySet()) {
+            ICacheKey<String> iCacheKey = getICacheKey(key);
             Boolean expectedOutput = expectedOutputs.get(key);
-            String value = tieredSpilloverCache.computeIfAbsent(key, loader);
+            String value = tieredSpilloverCache.computeIfAbsent(iCacheKey, loader);
             assertEquals(keyValuePairs.get(key), value);
-            String result = tieredSpilloverCache.get(key);
+            String result = tieredSpilloverCache.get(iCacheKey);
             if (expectedOutput) {
                 // Should retrieve from disk tier if it was accepted
                 assertEquals(keyValuePairs.get(key), result);
@@ -959,9 +1004,7 @@ public void testTookTimePolicyFromFactory() throws Exception {
                 onHeapCacheSize * keyValueSize + "b"
             )
             .put(
-                TieredSpilloverCacheSettings.TIERED_SPILLOVER_DISK_TOOK_TIME_THRESHOLD.getConcreteSettingForNamespace(
-                    CacheType.INDICES_REQUEST_CACHE.getSettingPrefix()
-                ).getKey(),
+                TieredSpilloverCacheSettings.TOOK_TIME_POLICY_CONCRETE_SETTINGS_MAP.get(CacheType.INDICES_REQUEST_CACHE).getKey(),
                 new TimeValue(timeValueThresholdNanos / 1_000_000)
             )
             .build();
@@ -973,19 +1016,21 @@ public void testTookTimePolicyFromFactory() throws Exception {
                 .setRemovalListener(removalListener)
                 .setSettings(settings)
                 .setMaxSizeInBytes(onHeapCacheSize * keyValueSize)
+                .setDimensionNames(dimensionNames)
                 .setCachedResultParser(new Function<String, CachedQueryResult.PolicyValues>() {
                     @Override
                     public CachedQueryResult.PolicyValues apply(String s) {
                         return new CachedQueryResult.PolicyValues(tookTimeMap.get(s));
                     }
                 })
+                .setClusterSettings(clusterSettings)
                 .build(),
             CacheType.INDICES_REQUEST_CACHE,
             Map.of(
                 OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory.NAME,
                 new OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory(),
                 MockDiskCache.MockDiskCacheFactory.NAME,
-                new MockDiskCache.MockDiskCacheFactory(0, randomIntBetween(100, 300))
+                new MockDiskCache.MockDiskCacheFactory(0, randomIntBetween(100, 300), false)
             )
         );
 
@@ -993,22 +1038,22 @@ public CachedQueryResult.PolicyValues apply(String s) {
 
         // First add all our values to the on heap cache
         for (String key : tookTimeMap.keySet()) {
-            tieredSpilloverCache.computeIfAbsent(key, getLoadAwareCacheLoader(keyValueMap));
+            tieredSpilloverCache.computeIfAbsent(getICacheKey(key), getLoadAwareCacheLoader(keyValueMap));
         }
         assertEquals(tookTimeMap.size(), tieredSpilloverCache.count());
 
         // Ensure all these keys get evicted from the on heap tier by adding > heap tier size worth of random keys
         for (int i = 0; i < onHeapCacheSize; i++) {
-            tieredSpilloverCache.computeIfAbsent(UUID.randomUUID().toString(), getLoadAwareCacheLoader(keyValueMap));
+            tieredSpilloverCache.computeIfAbsent(getICacheKey(UUID.randomUUID().toString()), getLoadAwareCacheLoader(keyValueMap));
         }
         ICache<String, String> onHeapCache = tieredSpilloverCache.getOnHeapCache();
         for (String key : tookTimeMap.keySet()) {
-            assertNull(onHeapCache.get(key));
+            assertNull(onHeapCache.get(getICacheKey(key)));
         }
 
         // Now the original keys should be in the disk tier if the policy allows them, or misses if not
         for (String key : tookTimeMap.keySet()) {
-            String computedValue = tieredSpilloverCache.get(key);
+            String computedValue = tieredSpilloverCache.get(getICacheKey(key));
             String mapValue = keyValueMap.get(key);
             Long tookTime = tookTimeMap.get(mapValue);
             if (tookTime != null && tookTime > timeValueThresholdNanos) {
@@ -1024,8 +1069,9 @@ public CachedQueryResult.PolicyValues apply(String s) {
     public void testMinimumThresholdSettingValue() throws Exception {
         // Confirm we can't set TieredSpilloverCache.TieredSpilloverCacheFactory.TIERED_SPILLOVER_DISK_TOOK_TIME_THRESHOLD to below
         // TimeValue.ZERO (for example, MINUS_ONE)
-        Setting<TimeValue> concreteSetting = TieredSpilloverCacheSettings.TIERED_SPILLOVER_DISK_TOOK_TIME_THRESHOLD
-            .getConcreteSettingForNamespace(CacheType.INDICES_REQUEST_CACHE.getSettingPrefix());
+        Setting<TimeValue> concreteSetting = TieredSpilloverCacheSettings.TOOK_TIME_POLICY_CONCRETE_SETTINGS_MAP.get(
+            CacheType.INDICES_REQUEST_CACHE
+        );
         TimeValue validDuration = new TimeValue(0, TimeUnit.MILLISECONDS);
         Settings validSettings = Settings.builder().put(concreteSetting.getKey(), validDuration).build();
 
@@ -1035,6 +1081,258 @@ public void testMinimumThresholdSettingValue() throws Exception {
         assertEquals(validDuration, concreteSetting.get(validSettings));
     }
 
+    public void testPutWithDiskCacheDisabledSetting() throws Exception {
+        int onHeapCacheSize = randomIntBetween(10, 30);
+        int diskCacheSize = randomIntBetween(onHeapCacheSize + 1, 100);
+        int keyValueSize = 50;
+        int totalSize = onHeapCacheSize + diskCacheSize;
+
+        MockCacheRemovalListener<String, String> removalListener = new MockCacheRemovalListener<>();
+        TieredSpilloverCache<String, String> tieredSpilloverCache = initializeTieredSpilloverCache(
+            keyValueSize,
+            diskCacheSize,
+            removalListener,
+            Settings.builder()
+                .put(
+                    OpenSearchOnHeapCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE)
+                        .get(MAXIMUM_SIZE_IN_BYTES_KEY)
+                        .getKey(),
+                    onHeapCacheSize * keyValueSize + "b"
+                )
+                .put(DISK_CACHE_ENABLED_SETTING_MAP.get(CacheType.INDICES_REQUEST_CACHE).getKey(), false)
+                .build(),
+            0
+        );
+
+        int numOfItems1 = randomIntBetween(onHeapCacheSize + 1, totalSize); // Create more items than onHeap cache.
+        for (int iter = 0; iter < numOfItems1; iter++) {
+            ICacheKey<String> key = getICacheKey(UUID.randomUUID().toString());
+            LoadAwareCacheLoader<ICacheKey<String>, String> loadAwareCacheLoader = getLoadAwareCacheLoader();
+            tieredSpilloverCache.computeIfAbsent(key, loadAwareCacheLoader);
+        }
+        ICache<String, String> onHeapCache = tieredSpilloverCache.getOnHeapCache();
+        ICache<String, String> diskCache = tieredSpilloverCache.getDiskCache();
+        assertEquals(onHeapCacheSize, onHeapCache.count());
+        assertEquals(0, diskCache.count()); // Disk cache shouldn't have anything considering it is disabled.
+        assertEquals(numOfItems1 - onHeapCacheSize, removalListener.evictionsMetric.count());
+    }
+
+    public void testGetPutAndInvalidateWithDiskCacheDisabled() throws Exception {
+        int onHeapCacheSize = randomIntBetween(10, 30);
+        int diskCacheSize = randomIntBetween(onHeapCacheSize + 1, 100);
+        int keyValueSize = 50;
+        int totalSize = onHeapCacheSize + diskCacheSize;
+        MockCacheRemovalListener<String, String> removalListener = new MockCacheRemovalListener<>();
+        TieredSpilloverCache<String, String> tieredSpilloverCache = initializeTieredSpilloverCache(
+            keyValueSize,
+            diskCacheSize,
+            removalListener,
+            Settings.builder()
+                .put(
+                    OpenSearchOnHeapCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE)
+                        .get(MAXIMUM_SIZE_IN_BYTES_KEY)
+                        .getKey(),
+                    onHeapCacheSize * keyValueSize + "b"
+                )
+                .build(),
+            0
+        );
+
+        int numOfItems1 = randomIntBetween(onHeapCacheSize + 1, totalSize - 1); // Create more items than onHeap
+        // cache to cause spillover.
+        for (int iter = 0; iter < numOfItems1; iter++) {
+            ICacheKey<String> key = getICacheKey(UUID.randomUUID().toString());
+            LoadAwareCacheLoader<ICacheKey<String>, String> loadAwareCacheLoader = getLoadAwareCacheLoader();
+            tieredSpilloverCache.computeIfAbsent(key, loadAwareCacheLoader);
+        }
+        ICache<String, String> onHeapCache = tieredSpilloverCache.getOnHeapCache();
+        ICache<String, String> diskCache = tieredSpilloverCache.getDiskCache();
+        List<ICacheKey<String>> diskCacheKeys = new ArrayList<>();
+        tieredSpilloverCache.getDiskCache().keys().forEach(diskCacheKeys::add);
+        long actualDiskCacheCount = diskCache.count();
+        long actualTieredCacheCount = tieredSpilloverCache.count();
+        assertEquals(onHeapCacheSize, onHeapCache.count());
+        assertEquals(numOfItems1 - onHeapCacheSize, actualDiskCacheCount);
+        assertEquals(0, removalListener.evictionsMetric.count());
+        assertEquals(numOfItems1, actualTieredCacheCount);
+        for (ICacheKey<String> diskKey : diskCacheKeys) {
+            assertNotNull(tieredSpilloverCache.get(diskKey));
+        }
+
+        tieredSpilloverCache.enableDisableDiskCache(false); // Disable disk cache now.
+        int numOfItems2 = totalSize - numOfItems1;
+        for (int iter = 0; iter < numOfItems2; iter++) {
+            ICacheKey<String> key = getICacheKey(UUID.randomUUID().toString());
+            LoadAwareCacheLoader<ICacheKey<String>, String> loadAwareCacheLoader = getLoadAwareCacheLoader();
+            tieredSpilloverCache.computeIfAbsent(key, loadAwareCacheLoader);
+        }
+        for (ICacheKey<String> diskKey : diskCacheKeys) {
+            assertNull(tieredSpilloverCache.get(diskKey)); // Considering disk cache is disabled, we shouldn't find
+            // these keys.
+        }
+        assertEquals(onHeapCacheSize, onHeapCache.count()); // Should remain same.
+        assertEquals(0, diskCache.count() - actualDiskCacheCount); // Considering it is disabled now, shouldn't cache
+        // any more items.
+        assertEquals(numOfItems2, removalListener.evictionsMetric.count()); // Considering onHeap cache was already
+        // full, we should all existing onHeap entries being evicted.
+        assertEquals(0, tieredSpilloverCache.count() - actualTieredCacheCount); // Count still returns disk cache
+        // entries count as they haven't been cleared yet.
+        long lastKnownTieredCacheEntriesCount = tieredSpilloverCache.count();
+
+        // Clear up disk cache keys.
+        for (ICacheKey<String> diskKey : diskCacheKeys) {
+            tieredSpilloverCache.invalidate(diskKey);
+        }
+        assertEquals(0, diskCache.count());
+        assertEquals(lastKnownTieredCacheEntriesCount - diskCacheKeys.size(), tieredSpilloverCache.count());
+
+        tieredSpilloverCache.invalidateAll(); // Clear up all the keys.
+        assertEquals(0, tieredSpilloverCache.count());
+    }
+
+    public void testTiersDoNotTrackStats() throws Exception {
+        int onHeapCacheSize = randomIntBetween(10, 30);
+        int diskCacheSize = randomIntBetween(onHeapCacheSize + 1, 100);
+        int keyValueSize = 50;
+        MockCacheRemovalListener<String, String> removalListener = new MockCacheRemovalListener<>();
+        TieredSpilloverCache<String, String> tieredSpilloverCache = initializeTieredSpilloverCache(
+            keyValueSize,
+            diskCacheSize,
+            removalListener,
+            Settings.builder()
+                .put(
+                    OpenSearchOnHeapCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE)
+                        .get(MAXIMUM_SIZE_IN_BYTES_KEY)
+                        .getKey(),
+                    onHeapCacheSize * keyValueSize + "b"
+                )
+                .build(),
+            0
+        );
+
+        // do some gets to put entries in both tiers
+        int numMisses = onHeapCacheSize + randomIntBetween(10, 20);
+        for (int iter = 0; iter < numMisses; iter++) {
+            ICacheKey<String> key = getICacheKey(UUID.randomUUID().toString());
+            LoadAwareCacheLoader<ICacheKey<String>, String> tieredCacheLoader = getLoadAwareCacheLoader();
+            tieredSpilloverCache.computeIfAbsent(key, tieredCacheLoader);
+        }
+        assertNotEquals(new ImmutableCacheStats(0, 0, 0, 0, 0), tieredSpilloverCache.stats().getTotalStats());
+        assertEquals(new ImmutableCacheStats(0, 0, 0, 0, 0), tieredSpilloverCache.getOnHeapCache().stats().getTotalStats());
+        ImmutableCacheStats diskStats = tieredSpilloverCache.getDiskCache().stats().getTotalStats();
+        assertEquals(new ImmutableCacheStats(0, 0, 0, 0, 0), diskStats);
+    }
+
+    public void testTierStatsAddCorrectly() throws Exception {
+        /* We expect the total stats to be:
+         * totalHits = heapHits + diskHits
+         * totalMisses = diskMisses
+         * totalEvictions = diskEvictions
+         * totalSize = heapSize + diskSize
+         * totalEntries = heapEntries + diskEntries
+         */
+
+        int onHeapCacheSize = randomIntBetween(10, 30);
+        int diskCacheSize = randomIntBetween(onHeapCacheSize + 1, 100);
+        int keyValueSize = 50;
+        MockCacheRemovalListener<String, String> removalListener = new MockCacheRemovalListener<>();
+        TieredSpilloverCache<String, String> tieredSpilloverCache = initializeTieredSpilloverCache(
+            keyValueSize,
+            diskCacheSize,
+            removalListener,
+            Settings.builder()
+                .put(
+                    OpenSearchOnHeapCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE)
+                        .get(MAXIMUM_SIZE_IN_BYTES_KEY)
+                        .getKey(),
+                    onHeapCacheSize * keyValueSize + "b"
+                )
+                .build(),
+            0
+        );
+
+        List<ICacheKey<String>> usedKeys = new ArrayList<>();
+        // Fill the cache, getting some entries + evictions for both tiers
+        int numMisses = onHeapCacheSize + diskCacheSize + randomIntBetween(10, 20);
+        for (int iter = 0; iter < numMisses; iter++) {
+            ICacheKey<String> key = getICacheKey(UUID.randomUUID().toString());
+            usedKeys.add(key);
+            LoadAwareCacheLoader<ICacheKey<String>, String> tieredCacheLoader = getLoadAwareCacheLoader();
+            tieredSpilloverCache.computeIfAbsent(key, tieredCacheLoader);
+        }
+        // Also do some random hits
+        Random rand = Randomness.get();
+        int approxNumHits = 30;
+        for (int i = 0; i < approxNumHits; i++) {
+            LoadAwareCacheLoader<ICacheKey<String>, String> tieredCacheLoader = getLoadAwareCacheLoader();
+            ICacheKey<String> key = usedKeys.get(rand.nextInt(usedKeys.size()));
+            tieredSpilloverCache.computeIfAbsent(key, tieredCacheLoader);
+        }
+
+        ImmutableCacheStats totalStats = tieredSpilloverCache.stats().getTotalStats();
+        ImmutableCacheStats heapStats = getStatsSnapshotForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP);
+        ImmutableCacheStats diskStats = getStatsSnapshotForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_DISK);
+
+        assertEquals(totalStats.getHits(), heapStats.getHits() + diskStats.getHits());
+        assertEquals(totalStats.getMisses(), diskStats.getMisses());
+        assertEquals(totalStats.getEvictions(), diskStats.getEvictions());
+        assertEquals(totalStats.getSizeInBytes(), heapStats.getSizeInBytes() + diskStats.getSizeInBytes());
+        assertEquals(totalStats.getItems(), heapStats.getItems() + diskStats.getItems());
+
+        // Also check the heap stats don't have zero misses or evictions
+        assertNotEquals(0, heapStats.getMisses());
+        assertNotEquals(0, heapStats.getEvictions());
+
+        // Now turn off the disk tier and do more misses and evictions from the heap tier.
+        // These should be added to the totals, as the disk tier is now absent
+        long missesBeforeDisablingDiskCache = totalStats.getMisses();
+        long evictionsBeforeDisablingDiskCache = totalStats.getEvictions();
+        long heapTierEvictionsBeforeDisablingDiskCache = heapStats.getEvictions();
+
+        clusterSettings.applySettings(
+            Settings.builder().put(DISK_CACHE_ENABLED_SETTING_MAP.get(CacheType.INDICES_REQUEST_CACHE).getKey(), false).build()
+        );
+
+        int newMisses = randomIntBetween(10, 30);
+        for (int i = 0; i < newMisses; i++) {
+            LoadAwareCacheLoader<ICacheKey<String>, String> tieredCacheLoader = getLoadAwareCacheLoader();
+            tieredSpilloverCache.computeIfAbsent(getICacheKey(UUID.randomUUID().toString()), tieredCacheLoader);
+        }
+
+        totalStats = tieredSpilloverCache.stats().getTotalStats();
+        heapStats = getStatsSnapshotForTier(tieredSpilloverCache, TIER_DIMENSION_VALUE_ON_HEAP);
+        assertEquals(missesBeforeDisablingDiskCache + newMisses, totalStats.getMisses());
+        assertEquals(heapTierEvictionsBeforeDisablingDiskCache + newMisses, heapStats.getEvictions());
+        assertEquals(evictionsBeforeDisablingDiskCache + newMisses, totalStats.getEvictions());
+
+        // Turn the disk cache back on in cluster settings for other tests
+        clusterSettings.applySettings(
+            Settings.builder().put(DISK_CACHE_ENABLED_SETTING_MAP.get(CacheType.INDICES_REQUEST_CACHE).getKey(), true).build()
+        );
+
+    }
+
+    private List<String> getMockDimensions() {
+        List<String> dims = new ArrayList<>();
+        for (String dimensionName : dimensionNames) {
+            dims.add("0");
+        }
+        return dims;
+    }
+
+    private ICacheKey<String> getICacheKey(String key) {
+        return new ICacheKey<>(key, getMockDimensions());
+    }
+
+    class MockCacheRemovalListener<K, V> implements RemovalListener<ICacheKey<K>, V> {
+        final CounterMetric evictionsMetric = new CounterMetric();
+
+        @Override
+        public void onRemoval(RemovalNotification<ICacheKey<K>, V> notification) {
+            evictionsMetric.inc();
+        }
+    }
+
     private static class AllowFirstLetterA implements Predicate<String> {
         @Override
         public boolean test(String data) {
@@ -1053,12 +1351,12 @@ public boolean test(String data) {
         }
     }
 
-    private LoadAwareCacheLoader<String, String> getLoadAwareCacheLoader() {
+    private LoadAwareCacheLoader<ICacheKey<String>, String> getLoadAwareCacheLoader() {
         return new LoadAwareCacheLoader<>() {
             boolean isLoaded = false;
 
             @Override
-            public String load(String key) {
+            public String load(ICacheKey<String> key) {
                 isLoaded = true;
                 return UUID.randomUUID().toString();
             }
@@ -1070,14 +1368,14 @@ public boolean isLoaded() {
         };
     }
 
-    private LoadAwareCacheLoader<String, String> getLoadAwareCacheLoader(Map<String, String> keyValueMap) {
+    private LoadAwareCacheLoader<ICacheKey<String>, String> getLoadAwareCacheLoader(Map<String, String> keyValueMap) {
         return new LoadAwareCacheLoader<>() {
             boolean isLoaded = false;
 
             @Override
-            public String load(String key) {
+            public String load(ICacheKey<String> key) {
                 isLoaded = true;
-                String mapValue = keyValueMap.get(key);
+                String mapValue = keyValueMap.get(key.key);
                 if (mapValue == null) {
                     mapValue = UUID.randomUUID().toString();
                 }
@@ -1091,10 +1389,10 @@ public boolean isLoaded() {
         };
     }
 
-    private TieredSpilloverCache<String, String> intializeTieredSpilloverCache(
+    private TieredSpilloverCache<String, String> initializeTieredSpilloverCache(
         int keyValueSize,
         int diskCacheSize,
-        RemovalListener<String, String> removalListener,
+        RemovalListener<ICacheKey<String>, String> removalListener,
         Settings settings,
         long diskDeliberateDelay
 
@@ -1105,7 +1403,7 @@ private TieredSpilloverCache<String, String> intializeTieredSpilloverCache(
     private TieredSpilloverCache<String, String> intializeTieredSpilloverCache(
         int keyValueSize,
         int diskCacheSize,
-        RemovalListener<String, String> removalListener,
+        RemovalListener<ICacheKey<String>, String> removalListener,
         Settings settings,
         long diskDeliberateDelay,
         List<Predicate<String>> policies
@@ -1114,6 +1412,8 @@ private TieredSpilloverCache<String, String> intializeTieredSpilloverCache(
         CacheConfig<String, String> cacheConfig = new CacheConfig.Builder<String, String>().setKeyType(String.class)
             .setKeyType(String.class)
             .setWeigher((k, v) -> keyValueSize)
+            .setSettings(settings)
+            .setDimensionNames(dimensionNames)
             .setRemovalListener(removalListener)
             .setSettings(
                 Settings.builder()
@@ -1125,8 +1425,9 @@ private TieredSpilloverCache<String, String> intializeTieredSpilloverCache(
                     .put(settings)
                     .build()
             )
+            .setClusterSettings(clusterSettings)
             .build();
-        ICache.Factory mockDiskCacheFactory = new MockDiskCache.MockDiskCacheFactory(diskDeliberateDelay, diskCacheSize);
+        ICache.Factory mockDiskCacheFactory = new MockDiskCache.MockDiskCacheFactory(diskDeliberateDelay, diskCacheSize, false);
 
         TieredSpilloverCache.Builder<String, String> builder = new TieredSpilloverCache.Builder<String, String>().setCacheType(
             CacheType.INDICES_REQUEST_CACHE
@@ -1140,4 +1441,42 @@ private TieredSpilloverCache<String, String> intializeTieredSpilloverCache(
         }
         return builder.build();
     }
+
+    // Helper functions for extracting tier aggregated stats.
+    private long getHitsForTier(TieredSpilloverCache<?, ?> tsc, String tierValue) throws IOException {
+        return getStatsSnapshotForTier(tsc, tierValue).getHits();
+    }
+
+    private long getMissesForTier(TieredSpilloverCache<?, ?> tsc, String tierValue) throws IOException {
+        return getStatsSnapshotForTier(tsc, tierValue).getMisses();
+    }
+
+    private long getEvictionsForTier(TieredSpilloverCache<?, ?> tsc, String tierValue) throws IOException {
+        return getStatsSnapshotForTier(tsc, tierValue).getEvictions();
+    }
+
+    private long getSizeInBytesForTier(TieredSpilloverCache<?, ?> tsc, String tierValue) throws IOException {
+        return getStatsSnapshotForTier(tsc, tierValue).getSizeInBytes();
+    }
+
+    private long getItemsForTier(TieredSpilloverCache<?, ?> tsc, String tierValue) throws IOException {
+        return getStatsSnapshotForTier(tsc, tierValue).getItems();
+    }
+
+    private ImmutableCacheStats getStatsSnapshotForTier(TieredSpilloverCache<?, ?> tsc, String tierValue) throws IOException {
+        List<String> levelsList = new ArrayList<>(dimensionNames);
+        levelsList.add(TIER_DIMENSION_NAME);
+        String[] levels = levelsList.toArray(new String[0]);
+        ImmutableCacheStatsHolder cacheStats = tsc.stats(levels);
+        // Since we always use the same list of dimensions from getMockDimensions() in keys for these tests, we can get all the stats values
+        // for a given tier with a single node in MDCS
+        List<String> mockDimensions = getMockDimensions();
+        mockDimensions.add(tierValue);
+        ImmutableCacheStats snapshot = cacheStats.getStatsForDimensionValues(mockDimensions);
+        if (snapshot == null) {
+            return new ImmutableCacheStats(0, 0, 0, 0, 0); // This can happen if no cache actions have happened for this set of
+            // dimensions yet
+        }
+        return snapshot;
+    }
 }
diff --git a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/ConvertProcessor.java b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/ConvertProcessor.java
index 2a81fa5f4986e..c7b5a8978188f 100644
--- a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/ConvertProcessor.java
+++ b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/ConvertProcessor.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.ingest.common;
 
+import org.opensearch.common.network.InetAddresses;
 import org.opensearch.ingest.AbstractProcessor;
 import org.opensearch.ingest.ConfigurationUtils;
 import org.opensearch.ingest.IngestDocument;
@@ -118,6 +119,19 @@ public Object convert(Object value) {
                 return value.toString();
             }
         },
+        IP {
+            @Override
+            public Object convert(Object value) {
+                // If the value is a valid ipv4/ipv6 address, we return the original value directly because IpFieldType
+                // can accept string value, this is simpler than we return an InetAddress object which needs to do more
+                // work such as serialization
+                if (value instanceof String && InetAddresses.isInetAddress(value.toString())) {
+                    return value;
+                } else {
+                    throw new IllegalArgumentException("[" + value + "] is not a valid ipv4/ipv6 address");
+                }
+            }
+        },
         AUTO {
             @Override
             public Object convert(Object value) {
diff --git a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/RenameProcessor.java b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/RenameProcessor.java
index 7564bbdf95f45..6ec3ebb6ace81 100644
--- a/modules/ingest-common/src/main/java/org/opensearch/ingest/common/RenameProcessor.java
+++ b/modules/ingest-common/src/main/java/org/opensearch/ingest/common/RenameProcessor.java
@@ -52,18 +52,21 @@ public final class RenameProcessor extends AbstractProcessor {
     private final TemplateScript.Factory field;
     private final TemplateScript.Factory targetField;
     private final boolean ignoreMissing;
+    private final boolean overrideTarget;
 
     RenameProcessor(
         String tag,
         String description,
         TemplateScript.Factory field,
         TemplateScript.Factory targetField,
-        boolean ignoreMissing
+        boolean ignoreMissing,
+        boolean overrideTarget
     ) {
         super(tag, description);
         this.field = field;
         this.targetField = targetField;
         this.ignoreMissing = ignoreMissing;
+        this.overrideTarget = overrideTarget;
     }
 
     TemplateScript.Factory getField() {
@@ -78,6 +81,10 @@ boolean isIgnoreMissing() {
         return ignoreMissing;
     }
 
+    boolean isOverrideTarget() {
+        return overrideTarget;
+    }
+
     @Override
     public IngestDocument execute(IngestDocument document) {
         String path = document.renderTemplate(field);
@@ -94,9 +101,10 @@ public IngestDocument execute(IngestDocument document) {
         // We fail here if the target field point to an array slot that is out of range.
         // If we didn't do this then we would fail if we set the value in the target_field
         // and then on failure processors would not see that value we tried to rename as we already
-        // removed it.
+        // removed it. If the target field is out of range, we throw the exception no matter
+        // what the parameter overrideTarget is.
         String target = document.renderTemplate(targetField);
-        if (document.hasField(target, true)) {
+        if (document.hasField(target, true) && !overrideTarget) {
             throw new IllegalArgumentException("field [" + target + "] already exists");
         }
 
@@ -143,7 +151,8 @@ public RenameProcessor create(
                 scriptService
             );
             boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
-            return new RenameProcessor(processorTag, description, fieldTemplate, targetFieldTemplate, ignoreMissing);
+            boolean overrideTarget = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "override_target", false);
+            return new RenameProcessor(processorTag, description, fieldTemplate, targetFieldTemplate, ignoreMissing, overrideTarget);
         }
     }
 }
diff --git a/modules/ingest-common/src/test/java/org/opensearch/ingest/common/ConvertProcessorTests.java b/modules/ingest-common/src/test/java/org/opensearch/ingest/common/ConvertProcessorTests.java
index 0ba0a39261d00..50ece9282888f 100644
--- a/modules/ingest-common/src/test/java/org/opensearch/ingest/common/ConvertProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/opensearch/ingest/common/ConvertProcessorTests.java
@@ -550,4 +550,29 @@ public void testTargetField() throws Exception {
         assertThat(ingestDocument.getFieldValue(fieldName, String.class), equalTo(String.valueOf(randomInt)));
         assertThat(ingestDocument.getFieldValue(targetField, Integer.class), equalTo(randomInt));
     }
+
+    public void testConvertIP() throws Exception {
+        IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
+        String validIPString;
+        if (randomBoolean()) {
+            validIPString = "1.2.3.4";
+        } else {
+            validIPString = "::1";
+        }
+        String fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, validIPString);
+
+        Processor processor = new ConvertProcessor(randomAlphaOfLength(10), null, fieldName, fieldName, Type.IP, false);
+        processor.execute(ingestDocument);
+        assertThat(ingestDocument.getFieldValue(fieldName, String.class), equalTo(validIPString));
+
+        String invalidIPString = randomAlphaOfLength(10);
+        fieldName = RandomDocumentPicks.addRandomField(random(), ingestDocument, invalidIPString);
+        Processor processorWithInvalidIP = new ConvertProcessor(randomAlphaOfLength(10), null, fieldName, fieldName, Type.IP, false);
+        try {
+            processorWithInvalidIP.execute(ingestDocument);
+            fail("processor execute should have failed");
+        } catch (IllegalArgumentException e) {
+            assertThat(e.getMessage(), equalTo("[" + invalidIPString + "] is not a valid ipv4/ipv6 address"));
+        }
+    }
 }
diff --git a/modules/ingest-common/src/test/java/org/opensearch/ingest/common/CopyProcessorTests.java b/modules/ingest-common/src/test/java/org/opensearch/ingest/common/CopyProcessorTests.java
index 3259ba85ef340..b53ce2db994a8 100644
--- a/modules/ingest-common/src/test/java/org/opensearch/ingest/common/CopyProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/opensearch/ingest/common/CopyProcessorTests.java
@@ -24,7 +24,7 @@ public class CopyProcessorTests extends OpenSearchTestCase {
     public void testCopyExistingField() throws Exception {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
         String sourceFieldName = RandomDocumentPicks.randomExistingFieldName(random(), ingestDocument);
-        String targetFieldName = RandomDocumentPicks.randomFieldName(random());
+        String targetFieldName = RandomDocumentPicks.randomNonExistingFieldName(random(), ingestDocument);
         Processor processor = createCopyProcessor(sourceFieldName, targetFieldName, false, false, false);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.hasField(targetFieldName), equalTo(true));
@@ -71,7 +71,8 @@ public void testCopyWithIgnoreMissing() throws Exception {
     public void testCopyWithRemoveSource() throws Exception {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
         String sourceFieldName = RandomDocumentPicks.randomExistingFieldName(random(), ingestDocument);
-        String targetFieldName = RandomDocumentPicks.randomFieldName(random());
+        String targetFieldName = RandomDocumentPicks.randomNonExistingFieldName(random(), ingestDocument);
+
         Object sourceValue = ingestDocument.getFieldValue(sourceFieldName, Object.class);
 
         Processor processor = createCopyProcessor(sourceFieldName, targetFieldName, false, true, false);
diff --git a/modules/ingest-common/src/test/java/org/opensearch/ingest/common/DotExpanderProcessorTests.java b/modules/ingest-common/src/test/java/org/opensearch/ingest/common/DotExpanderProcessorTests.java
index cd912269a593d..73719b24c74ea 100644
--- a/modules/ingest-common/src/test/java/org/opensearch/ingest/common/DotExpanderProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/opensearch/ingest/common/DotExpanderProcessorTests.java
@@ -105,6 +105,7 @@ public void testEscapeFields_valueField() throws Exception {
             null,
             new TestTemplateService.MockTemplateScript.Factory("foo"),
             new TestTemplateService.MockTemplateScript.Factory("foo.bar"),
+            false,
             false
         );
         processor.execute(document);
diff --git a/modules/ingest-common/src/test/java/org/opensearch/ingest/common/RenameProcessorFactoryTests.java b/modules/ingest-common/src/test/java/org/opensearch/ingest/common/RenameProcessorFactoryTests.java
index ec43be97689ee..8ce9203db43ce 100644
--- a/modules/ingest-common/src/test/java/org/opensearch/ingest/common/RenameProcessorFactoryTests.java
+++ b/modules/ingest-common/src/test/java/org/opensearch/ingest/common/RenameProcessorFactoryTests.java
@@ -77,6 +77,19 @@ public void testCreateWithIgnoreMissing() throws Exception {
         assertThat(renameProcessor.isIgnoreMissing(), equalTo(true));
     }
 
+    public void testCreateWithOverrideTarget() throws Exception {
+        Map<String, Object> config = new HashMap<>();
+        config.put("field", "old_field");
+        config.put("target_field", "new_field");
+        config.put("override_target", true);
+        String processorTag = randomAlphaOfLength(10);
+        RenameProcessor renameProcessor = factory.create(null, processorTag, null, config);
+        assertThat(renameProcessor.getTag(), equalTo(processorTag));
+        assertThat(renameProcessor.getField().newInstance(Collections.emptyMap()).execute(), equalTo("old_field"));
+        assertThat(renameProcessor.getTargetField().newInstance(Collections.emptyMap()).execute(), equalTo("new_field"));
+        assertThat(renameProcessor.isOverrideTarget(), equalTo(true));
+    }
+
     public void testCreateNoFieldPresent() throws Exception {
         Map<String, Object> config = new HashMap<>();
         config.put("target_field", "new_field");
diff --git a/modules/ingest-common/src/test/java/org/opensearch/ingest/common/RenameProcessorTests.java b/modules/ingest-common/src/test/java/org/opensearch/ingest/common/RenameProcessorTests.java
index a600464371af8..ad5b46e924278 100644
--- a/modules/ingest-common/src/test/java/org/opensearch/ingest/common/RenameProcessorTests.java
+++ b/modules/ingest-common/src/test/java/org/opensearch/ingest/common/RenameProcessorTests.java
@@ -59,7 +59,7 @@ public void testRename() throws Exception {
         do {
             newFieldName = RandomDocumentPicks.randomFieldName(random());
         } while (RandomDocumentPicks.canAddField(newFieldName, ingestDocument) == false || newFieldName.equals(fieldName));
-        Processor processor = createRenameProcessor(fieldName, newFieldName, false);
+        Processor processor = createRenameProcessor(fieldName, newFieldName, false, false);
         processor.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue(newFieldName, Object.class), equalTo(fieldValue));
     }
@@ -77,7 +77,7 @@ public void testRenameArrayElement() throws Exception {
         document.put("one", one);
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), document);
 
-        Processor processor = createRenameProcessor("list.0", "item", false);
+        Processor processor = createRenameProcessor("list.0", "item", false, false);
         processor.execute(ingestDocument);
         Object actualObject = ingestDocument.getSourceAndMetadata().get("list");
         assertThat(actualObject, instanceOf(List.class));
@@ -90,7 +90,7 @@ public void testRenameArrayElement() throws Exception {
         assertThat(actualObject, instanceOf(String.class));
         assertThat(actualObject, equalTo("item1"));
 
-        processor = createRenameProcessor("list.0", "list.3", false);
+        processor = createRenameProcessor("list.0", "list.3", false, randomBoolean());
         try {
             processor.execute(ingestDocument);
             fail("processor execute should have failed");
@@ -105,7 +105,7 @@ public void testRenameArrayElement() throws Exception {
     public void testRenameNonExistingField() throws Exception {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
         String fieldName = RandomDocumentPicks.randomFieldName(random());
-        Processor processor = createRenameProcessor(fieldName, RandomDocumentPicks.randomFieldName(random()), false);
+        Processor processor = createRenameProcessor(fieldName, RandomDocumentPicks.randomFieldName(random()), false, false);
         try {
             processor.execute(ingestDocument);
             fail("processor execute should have failed");
@@ -114,7 +114,7 @@ public void testRenameNonExistingField() throws Exception {
         }
 
         // when using template snippet, the resolved field path maybe empty
-        processor = createRenameProcessor("", RandomDocumentPicks.randomFieldName(random()), false);
+        processor = createRenameProcessor("", RandomDocumentPicks.randomFieldName(random()), false, false);
         try {
             processor.execute(ingestDocument);
             fail("processor execute should have failed");
@@ -127,30 +127,36 @@ public void testRenameNonExistingFieldWithIgnoreMissing() throws Exception {
         IngestDocument originalIngestDocument = RandomDocumentPicks.randomIngestDocument(random(), new HashMap<>());
         IngestDocument ingestDocument = new IngestDocument(originalIngestDocument);
         String fieldName = RandomDocumentPicks.randomFieldName(random());
-        Processor processor = createRenameProcessor(fieldName, RandomDocumentPicks.randomFieldName(random()), true);
+        Processor processor = createRenameProcessor(fieldName, RandomDocumentPicks.randomFieldName(random()), true, false);
         processor.execute(ingestDocument);
         assertIngestDocument(originalIngestDocument, ingestDocument);
 
         // when using template snippet, the resolved field path maybe empty
-        processor = createRenameProcessor("", RandomDocumentPicks.randomFieldName(random()), true);
+        processor = createRenameProcessor("", RandomDocumentPicks.randomFieldName(random()), true, false);
         processor.execute(ingestDocument);
         assertIngestDocument(originalIngestDocument, ingestDocument);
     }
 
     public void testRenameNewFieldAlreadyExists() throws Exception {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
-        String fieldName = RandomDocumentPicks.randomExistingFieldName(random(), ingestDocument);
-        Processor processor = createRenameProcessor(
-            RandomDocumentPicks.randomExistingFieldName(random(), ingestDocument),
-            fieldName,
-            false
-        );
+        String field = RandomDocumentPicks.randomExistingFieldName(random(), ingestDocument);
+        Object fieldValue = ingestDocument.getFieldValue(field, Object.class);
+        String targetField = RandomDocumentPicks.addRandomField(random(), ingestDocument, RandomDocumentPicks.randomFieldValue(random()));
+
+        Processor processor = createRenameProcessor(field, targetField, false, false);
         try {
             processor.execute(ingestDocument);
             fail("processor execute should have failed");
         } catch (IllegalArgumentException e) {
-            assertThat(e.getMessage(), equalTo("field [" + fieldName + "] already exists"));
+            assertThat(e.getMessage(), equalTo("field [" + targetField + "] already exists"));
         }
+
+        Processor processorWithOverrideTarget = createRenameProcessor(field, targetField, false, true);
+
+        processorWithOverrideTarget.execute(ingestDocument);
+        assertThat(ingestDocument.hasField(field), equalTo(false));
+        assertThat(ingestDocument.hasField(targetField), equalTo(true));
+        assertThat(ingestDocument.getFieldValue(targetField, Object.class), equalTo(fieldValue));
     }
 
     public void testRenameExistingFieldNullValue() throws Exception {
@@ -158,7 +164,7 @@ public void testRenameExistingFieldNullValue() throws Exception {
         String fieldName = RandomDocumentPicks.randomFieldName(random());
         ingestDocument.setFieldValue(fieldName, null);
         String newFieldName = randomValueOtherThanMany(ingestDocument::hasField, () -> RandomDocumentPicks.randomFieldName(random()));
-        Processor processor = createRenameProcessor(fieldName, newFieldName, false);
+        Processor processor = createRenameProcessor(fieldName, newFieldName, false, false);
         processor.execute(ingestDocument);
         if (newFieldName.startsWith(fieldName + '.')) {
             assertThat(ingestDocument.getFieldValue(fieldName, Object.class), instanceOf(Map.class));
@@ -182,7 +188,7 @@ public Object put(String key, Object value) {
         source.put("list", Collections.singletonList("item"));
 
         IngestDocument ingestDocument = new IngestDocument(source, Collections.emptyMap());
-        Processor processor = createRenameProcessor("list", "new_field", false);
+        Processor processor = createRenameProcessor("list", "new_field", false, false);
         try {
             processor.execute(ingestDocument);
             fail("processor execute should have failed");
@@ -206,7 +212,7 @@ public Object remove(Object key) {
         source.put("list", Collections.singletonList("item"));
 
         IngestDocument ingestDocument = new IngestDocument(source, Collections.emptyMap());
-        Processor processor = createRenameProcessor("list", "new_field", false);
+        Processor processor = createRenameProcessor("list", "new_field", false, false);
         try {
             processor.execute(ingestDocument);
             fail("processor execute should have failed");
@@ -221,12 +227,12 @@ public void testRenameLeafIntoBranch() throws Exception {
         Map<String, Object> source = new HashMap<>();
         source.put("foo", "bar");
         IngestDocument ingestDocument = new IngestDocument(source, Collections.emptyMap());
-        Processor processor1 = createRenameProcessor("foo", "foo.bar", false);
+        Processor processor1 = createRenameProcessor("foo", "foo.bar", false, false);
         processor1.execute(ingestDocument);
         assertThat(ingestDocument.getFieldValue("foo", Map.class), equalTo(Collections.singletonMap("bar", "bar")));
         assertThat(ingestDocument.getFieldValue("foo.bar", String.class), equalTo("bar"));
 
-        Processor processor2 = createRenameProcessor("foo.bar", "foo.bar.baz", false);
+        Processor processor2 = createRenameProcessor("foo.bar", "foo.bar.baz", false, false);
         processor2.execute(ingestDocument);
         assertThat(
             ingestDocument.getFieldValue("foo", Map.class),
@@ -236,18 +242,19 @@ public void testRenameLeafIntoBranch() throws Exception {
         assertThat(ingestDocument.getFieldValue("foo.bar.baz", String.class), equalTo("bar"));
 
         // for fun lets try to restore it (which don't allow today)
-        Processor processor3 = createRenameProcessor("foo.bar.baz", "foo", false);
+        Processor processor3 = createRenameProcessor("foo.bar.baz", "foo", false, false);
         Exception e = expectThrows(IllegalArgumentException.class, () -> processor3.execute(ingestDocument));
         assertThat(e.getMessage(), equalTo("field [foo] already exists"));
     }
 
-    private RenameProcessor createRenameProcessor(String field, String targetField, boolean ignoreMissing) {
+    private RenameProcessor createRenameProcessor(String field, String targetField, boolean ignoreMissing, boolean overrideTarget) {
         return new RenameProcessor(
             randomAlphaOfLength(10),
             null,
             new TestTemplateService.MockTemplateScript.Factory(field),
             new TestTemplateService.MockTemplateScript.Factory(targetField),
-            ignoreMissing
+            ignoreMissing,
+            overrideTarget
         );
     }
 }
diff --git a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/280_rename_processor.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/280_rename_processor.yml
index 96b2256bcc1dc..0ef658896ff0a 100644
--- a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/280_rename_processor.yml
+++ b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/280_rename_processor.yml
@@ -64,3 +64,42 @@ teardown:
         index: test
         id: 1
   - match: { _source.message: "foo bar baz" }
+
+---
+"Test rename processor with override_target":
+  - skip:
+      version: " - 2.13.99"
+      reason: "introduced in 2.14.0"
+  - do:
+      ingest.put_pipeline:
+        id: "my_pipeline"
+        body:  >
+          {
+            "description": "_description",
+            "processors": [
+              {
+                "rename" : {
+                  "field" : "foo",
+                  "target_field" : "bar",
+                  "override_target" : true
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: test
+        id: 1
+        pipeline: "my_pipeline"
+        body: {
+          foo: "foo",
+          bar: "bar"
+        }
+
+  - do:
+      get:
+        index: test
+        id: 1
+  - match: { _source: { "bar": "foo" } }
diff --git a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/330_convert_processor.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/330_convert_processor.yml
new file mode 100644
index 0000000000000..994ed225dd624
--- /dev/null
+++ b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/330_convert_processor.yml
@@ -0,0 +1,83 @@
+---
+teardown:
+  - do:
+      ingest.delete_pipeline:
+        id: "1"
+        ignore: 404
+
+---
+"Test convert processor with ip type":
+  - skip:
+      version: " - 2.13.99"
+      reason: "introduced in 2.14.0"
+  - do:
+      ingest.put_pipeline:
+        id: "1"
+        body:  >
+          {
+            "processors": [
+              {
+                "convert" : {
+                  "field" : "raw_ip",
+                  "type": "ip"
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      catch: /\[1.1.1.\] is not a valid ipv4\/ipv6 address/
+      index:
+        index: test
+        id: 1
+        pipeline: "1"
+        body: {
+          raw_ip: "1.1.1."
+        }
+
+  - do:
+      ingest.put_pipeline:
+        id: "1"
+        body:  >
+          {
+            "processors": [
+              {
+                "convert" : {
+                  "field" : "raw_ip",
+                  "target_field" : "ip_field",
+                  "type" : "ip",
+                  "ignore_failure" : true
+                }
+              }
+            ]
+          }
+  - match: { acknowledged: true }
+
+  - do:
+      index:
+        index: test
+        id: 1
+        pipeline: "1"
+        body: {
+          raw_ip: "1.1.1."
+        }
+  - do:
+      get:
+        index: test
+        id: 1
+  - match: { _source: { raw_ip: "1.1.1."} }
+
+  - do:
+      index:
+        index: test
+        id: 1
+        pipeline: "1"
+        body: {
+          raw_ip: "1.1.1.1"
+        }
+  - do:
+      get:
+        index: test
+        id: 1
+  - match: { _source: { raw_ip: "1.1.1.1", ip_field: "1.1.1.1"} }
diff --git a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/70_bulk.yml b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/70_bulk.yml
index 2dfa17174b139..edb7b77eb8d28 100644
--- a/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/70_bulk.yml
+++ b/modules/ingest-common/src/yamlRestTest/resources/rest-api-spec/test/ingest/70_bulk.yml
@@ -144,3 +144,113 @@ teardown:
 
   - is_false: _source.field1
   - match: {_source.field2: value2}
+
+# related issue: https://github.com/opensearch-project/OpenSearch/issues/12854
+---
+"Test bulk honors pipeline in update action with upsert":
+  - skip:
+      version: " - 2.13.99"
+      reason: "fixed in 2.14.0"
+
+  - do:
+      bulk:
+        refresh: true
+        body:
+          - '{"update": {"_index": "test_index", "_id": "test_id3", "pipeline": "pipeline1"}}'
+          - '{"upsert": {"f1": "v2", "f2": 47}, "doc": {"x": 1}}'
+
+  - match: { errors: false }
+  - match: { items.0.update.result: created }
+
+  - do:
+      get:
+        index: test_index
+        id: test_id3
+  - match: { _source: {"f1": "v2", "f2": 47, "field1": "value1"}}
+
+---
+"Test bulk API with batch enabled happy case":
+  - skip:
+      version: " - 2.13.99"
+      reason: "Added in 2.14.0"
+
+  - do:
+      bulk:
+        refresh: true
+        batch_size: 2
+        pipeline: "pipeline1"
+        body:
+          - '{"index": {"_index": "test_index", "_id": "test_id1"}}'
+          - '{"text": "text1"}'
+          - '{"index": {"_index": "test_index", "_id": "test_id2"}}'
+          - '{"text": "text2"}'
+          - '{"index": {"_index": "test_index", "_id": "test_id3"}}'
+          - '{"text": "text3"}'
+          - '{"index": {"_index": "test_index", "_id": "test_id4"}}'
+          - '{"text": "text4"}'
+          - '{"index": {"_index": "test_index", "_id": "test_id5", "pipeline": "pipeline2"}}'
+          - '{"text": "text5"}'
+          - '{"index": {"_index": "test_index", "_id": "test_id6", "pipeline": "pipeline2"}}'
+          - '{"text": "text6"}'
+
+  - match: { errors: false }
+
+  - do:
+      get:
+        index: test_index
+        id: test_id5
+  - match: { _source: {"text": "text5", "field2": "value2"}}
+
+  - do:
+      get:
+        index: test_index
+        id: test_id3
+  - match: { _source: { "text": "text3", "field1": "value1" } }
+
+---
+"Test bulk API with batch_size missing":
+  - skip:
+      version: " - 2.13.99"
+      reason: "Added in 2.14.0"
+
+  - do:
+      bulk:
+        refresh: true
+        pipeline: "pipeline1"
+        body:
+          - '{"index": {"_index": "test_index", "_id": "test_id1"}}'
+          - '{"text": "text1"}'
+          - '{"index": {"_index": "test_index", "_id": "test_id2"}}'
+          - '{"text": "text2"}'
+
+  - match: { errors: false }
+
+  - do:
+      get:
+        index: test_index
+        id: test_id1
+  - match: { _source: { "text": "text1", "field1": "value1" } }
+
+  - do:
+      get:
+        index: test_index
+        id: test_id2
+  - match: { _source: { "text": "text2", "field1": "value1" } }
+
+---
+"Test bulk API with invalid batch_size":
+  - skip:
+      version: " - 2.13.99"
+      reason: "Added in 2.14.0"
+
+  - do:
+      catch: bad_request
+      bulk:
+        refresh: true
+        batch_size: -1
+        pipeline: "pipeline1"
+        body:
+          - '{"index": {"_index": "test_index", "_id": "test_id1"}}'
+          - '{"text": "text1"}'
+          - '{"index": {"_index": "test_index", "_id": "test_id2"}}'
+          - '{"text": "text2"}'
diff --git a/modules/lang-expression/licenses/asm-9.6.jar.sha1 b/modules/lang-expression/licenses/asm-9.6.jar.sha1
deleted file mode 100644
index 2d9e6a9d3cfd6..0000000000000
--- a/modules/lang-expression/licenses/asm-9.6.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-aa205cf0a06dbd8e04ece91c0b37c3f5d567546a
\ No newline at end of file
diff --git a/modules/lang-expression/licenses/asm-9.7.jar.sha1 b/modules/lang-expression/licenses/asm-9.7.jar.sha1
new file mode 100644
index 0000000000000..84c9a9703af6d
--- /dev/null
+++ b/modules/lang-expression/licenses/asm-9.7.jar.sha1
@@ -0,0 +1 @@
+073d7b3086e14beb604ced229c302feff6449723
\ No newline at end of file
diff --git a/modules/lang-expression/licenses/asm-commons-9.6.jar.sha1 b/modules/lang-expression/licenses/asm-commons-9.6.jar.sha1
deleted file mode 100644
index a0814f495771f..0000000000000
--- a/modules/lang-expression/licenses/asm-commons-9.6.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f1a9e5508eff490744144565c47326c8648be309
\ No newline at end of file
diff --git a/modules/lang-expression/licenses/asm-commons-9.7.jar.sha1 b/modules/lang-expression/licenses/asm-commons-9.7.jar.sha1
new file mode 100644
index 0000000000000..1de4404e7d5d0
--- /dev/null
+++ b/modules/lang-expression/licenses/asm-commons-9.7.jar.sha1
@@ -0,0 +1 @@
+e86dda4696d3c185fcc95d8d311904e7ce38a53f
\ No newline at end of file
diff --git a/modules/lang-expression/licenses/asm-tree-9.6.jar.sha1 b/modules/lang-expression/licenses/asm-tree-9.6.jar.sha1
deleted file mode 100644
index 101eb03b4b736..0000000000000
--- a/modules/lang-expression/licenses/asm-tree-9.6.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-c0cdda9d211e965d2a4448aa3fd86110f2f8c2de
\ No newline at end of file
diff --git a/modules/lang-expression/licenses/asm-tree-9.7.jar.sha1 b/modules/lang-expression/licenses/asm-tree-9.7.jar.sha1
new file mode 100644
index 0000000000000..d4eeef6151272
--- /dev/null
+++ b/modules/lang-expression/licenses/asm-tree-9.7.jar.sha1
@@ -0,0 +1 @@
+e446a17b175bfb733b87c5c2560ccb4e57d69f1a
\ No newline at end of file
diff --git a/modules/lang-expression/licenses/lucene-expressions-9.11.0-snapshot-8a555eb.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.11.0-snapshot-8a555eb.jar.sha1
deleted file mode 100644
index 82a17e2b79290..0000000000000
--- a/modules/lang-expression/licenses/lucene-expressions-9.11.0-snapshot-8a555eb.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-00759eaff8f62b38ba66a05f26ab784c268908d3
\ No newline at end of file
diff --git a/modules/lang-expression/licenses/lucene-expressions-9.11.0-snapshot-fb97840.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-9.11.0-snapshot-fb97840.jar.sha1
new file mode 100644
index 0000000000000..f4da6e39aeeb8
--- /dev/null
+++ b/modules/lang-expression/licenses/lucene-expressions-9.11.0-snapshot-fb97840.jar.sha1
@@ -0,0 +1 @@
+ab914b48665f484315b78e4b6787aa42f5966bb6
\ No newline at end of file
diff --git a/modules/lang-mustache/build.gradle b/modules/lang-mustache/build.gradle
index 14eafd8d43e13..bcf5c07ea8c64 100644
--- a/modules/lang-mustache/build.gradle
+++ b/modules/lang-mustache/build.gradle
@@ -38,7 +38,7 @@ opensearchplugin {
 }
 
 dependencies {
-  api "com.github.spullara.mustache.java:compiler:0.9.10"
+  api "com.github.spullara.mustache.java:compiler:0.9.13"
 }
 
 restResources {
diff --git a/modules/lang-mustache/licenses/compiler-0.9.10.jar.sha1 b/modules/lang-mustache/licenses/compiler-0.9.10.jar.sha1
deleted file mode 100644
index 6336318c2ce1a..0000000000000
--- a/modules/lang-mustache/licenses/compiler-0.9.10.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-6111ae24e3be9ecbd75f5fe908583fc14b4f0174
\ No newline at end of file
diff --git a/modules/lang-mustache/licenses/compiler-0.9.13.jar.sha1 b/modules/lang-mustache/licenses/compiler-0.9.13.jar.sha1
new file mode 100644
index 0000000000000..70d53aac260eb
--- /dev/null
+++ b/modules/lang-mustache/licenses/compiler-0.9.13.jar.sha1
@@ -0,0 +1 @@
+60666500a7dce7a5d3e17c09b46ea6f037192bd5
\ No newline at end of file
diff --git a/modules/lang-mustache/src/yamlRestTest/resources/rest-api-spec/test/lang_mustache/20_render_search_template.yml b/modules/lang-mustache/src/yamlRestTest/resources/rest-api-spec/test/lang_mustache/20_render_search_template.yml
index 946b63a65d923..a0f6828e66d79 100644
--- a/modules/lang-mustache/src/yamlRestTest/resources/rest-api-spec/test/lang_mustache/20_render_search_template.yml
+++ b/modules/lang-mustache/src/yamlRestTest/resources/rest-api-spec/test/lang_mustache/20_render_search_template.yml
@@ -47,7 +47,7 @@
   - match: { template_output.aggs.my_terms.terms.field: "my_other_field" }
 
   - do:
-      catch: /Improperly.closed.variable.in.query-template/
+      catch: /Improperly.closed.variable:.my_value.in.query-template/
       render_search_template:
         body: { "source": { "query": { "match": { "text": "{{{my_value}}" } }, "aggs": { "my_terms": { "terms": { "field": "{{my_field}}" } } } }, "params": { "my_value": "bar", "my_field": "field1" } }
 ---
@@ -99,7 +99,7 @@
   - match: { template_output.size: 100 }
 
   - do:
-      catch: /Improperly.closed.variable.in.query-template/
+      catch: /Improperly.closed.variable:.my_value.in.query-template/
       render_search_template:
         body: { "source": "{ \"query\": { \"match\": { \"text\": \"{{{my_value}}\" } }, \"size\": {{my_size}} }", "params": { "my_value": "bar", "my_size": 100 } }
 
diff --git a/modules/lang-painless/licenses/asm-9.6.jar.sha1 b/modules/lang-painless/licenses/asm-9.6.jar.sha1
deleted file mode 100644
index 2d9e6a9d3cfd6..0000000000000
--- a/modules/lang-painless/licenses/asm-9.6.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-aa205cf0a06dbd8e04ece91c0b37c3f5d567546a
\ No newline at end of file
diff --git a/modules/lang-painless/licenses/asm-9.7.jar.sha1 b/modules/lang-painless/licenses/asm-9.7.jar.sha1
new file mode 100644
index 0000000000000..84c9a9703af6d
--- /dev/null
+++ b/modules/lang-painless/licenses/asm-9.7.jar.sha1
@@ -0,0 +1 @@
+073d7b3086e14beb604ced229c302feff6449723
\ No newline at end of file
diff --git a/modules/lang-painless/licenses/asm-analysis-9.6.jar.sha1 b/modules/lang-painless/licenses/asm-analysis-9.6.jar.sha1
deleted file mode 100644
index fa42ea1198165..0000000000000
--- a/modules/lang-painless/licenses/asm-analysis-9.6.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-9ce6c7b174bd997fc2552dff47964546bd7a5ec3
\ No newline at end of file
diff --git a/modules/lang-painless/licenses/asm-analysis-9.7.jar.sha1 b/modules/lang-painless/licenses/asm-analysis-9.7.jar.sha1
new file mode 100644
index 0000000000000..c7687adfeb990
--- /dev/null
+++ b/modules/lang-painless/licenses/asm-analysis-9.7.jar.sha1
@@ -0,0 +1 @@
+e4a258b7eb96107106c0599f0061cfc1832fe07a
\ No newline at end of file
diff --git a/modules/lang-painless/licenses/asm-commons-9.6.jar.sha1 b/modules/lang-painless/licenses/asm-commons-9.6.jar.sha1
deleted file mode 100644
index a0814f495771f..0000000000000
--- a/modules/lang-painless/licenses/asm-commons-9.6.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f1a9e5508eff490744144565c47326c8648be309
\ No newline at end of file
diff --git a/modules/lang-painless/licenses/asm-commons-9.7.jar.sha1 b/modules/lang-painless/licenses/asm-commons-9.7.jar.sha1
new file mode 100644
index 0000000000000..1de4404e7d5d0
--- /dev/null
+++ b/modules/lang-painless/licenses/asm-commons-9.7.jar.sha1
@@ -0,0 +1 @@
+e86dda4696d3c185fcc95d8d311904e7ce38a53f
\ No newline at end of file
diff --git a/modules/lang-painless/licenses/asm-tree-9.6.jar.sha1 b/modules/lang-painless/licenses/asm-tree-9.6.jar.sha1
deleted file mode 100644
index 101eb03b4b736..0000000000000
--- a/modules/lang-painless/licenses/asm-tree-9.6.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-c0cdda9d211e965d2a4448aa3fd86110f2f8c2de
\ No newline at end of file
diff --git a/modules/lang-painless/licenses/asm-tree-9.7.jar.sha1 b/modules/lang-painless/licenses/asm-tree-9.7.jar.sha1
new file mode 100644
index 0000000000000..d4eeef6151272
--- /dev/null
+++ b/modules/lang-painless/licenses/asm-tree-9.7.jar.sha1
@@ -0,0 +1 @@
+e446a17b175bfb733b87c5c2560ccb4e57d69f1a
\ No newline at end of file
diff --git a/modules/lang-painless/licenses/asm-util-9.6.jar.sha1 b/modules/lang-painless/licenses/asm-util-9.6.jar.sha1
deleted file mode 100644
index 1f42ac62dc69c..0000000000000
--- a/modules/lang-painless/licenses/asm-util-9.6.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f77caf84eb93786a749b2baa40865b9613e3eaee
\ No newline at end of file
diff --git a/modules/lang-painless/licenses/asm-util-9.7.jar.sha1 b/modules/lang-painless/licenses/asm-util-9.7.jar.sha1
new file mode 100644
index 0000000000000..37c0d27efe46f
--- /dev/null
+++ b/modules/lang-painless/licenses/asm-util-9.7.jar.sha1
@@ -0,0 +1 @@
+c0655519f24d92af2202cb681cd7c1569df6ead6
\ No newline at end of file
diff --git a/modules/lang-painless/src/main/java/org/opensearch/painless/PainlessModulePlugin.java b/modules/lang-painless/src/main/java/org/opensearch/painless/PainlessModulePlugin.java
index c7638b3c41c63..55dc23f665d2e 100644
--- a/modules/lang-painless/src/main/java/org/opensearch/painless/PainlessModulePlugin.java
+++ b/modules/lang-painless/src/main/java/org/opensearch/painless/PainlessModulePlugin.java
@@ -60,6 +60,7 @@
 import org.opensearch.repositories.RepositoriesService;
 import org.opensearch.rest.RestController;
 import org.opensearch.rest.RestHandler;
+import org.opensearch.script.DerivedFieldScript;
 import org.opensearch.script.IngestScript;
 import org.opensearch.script.ScoreScript;
 import org.opensearch.script.ScriptContext;
@@ -108,6 +109,11 @@ public final class PainlessModulePlugin extends Plugin implements ScriptPlugin,
         ingest.add(AllowlistLoader.loadFromResourceFiles(Allowlist.class, "org.opensearch.ingest.txt"));
         map.put(IngestScript.CONTEXT, ingest);
 
+        // Functions available to derived fields
+        List<Allowlist> derived = new ArrayList<>(Allowlist.BASE_ALLOWLISTS);
+        derived.add(AllowlistLoader.loadFromResourceFiles(Allowlist.class, "org.opensearch.derived.txt"));
+        map.put(DerivedFieldScript.CONTEXT, derived);
+
         allowlists = map;
     }
 
diff --git a/modules/lang-painless/src/main/resources/org/opensearch/painless/spi/org.opensearch.derived.txt b/modules/lang-painless/src/main/resources/org/opensearch/painless/spi/org.opensearch.derived.txt
new file mode 100644
index 0000000000000..9a3dd4894b286
--- /dev/null
+++ b/modules/lang-painless/src/main/resources/org/opensearch/painless/spi/org.opensearch.derived.txt
@@ -0,0 +1,17 @@
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+
+# This file contains an allowlist for functions to be used in derived field context
+
+class org.opensearch.script.DerivedFieldScript @no_import {
+}
+
+static_import {
+    void emit(org.opensearch.script.DerivedFieldScript, Object) bound_to org.opensearch.script.ScriptEmitValues$EmitSingle
+    void emit(org.opensearch.script.DerivedFieldScript, double, double) bound_to org.opensearch.script.ScriptEmitValues$GeoPoint
+}
diff --git a/modules/lang-painless/src/test/java/org/opensearch/painless/DerivedFieldScriptTests.java b/modules/lang-painless/src/test/java/org/opensearch/painless/DerivedFieldScriptTests.java
new file mode 100644
index 0000000000000..2340e5b238ebb
--- /dev/null
+++ b/modules/lang-painless/src/test/java/org/opensearch/painless/DerivedFieldScriptTests.java
@@ -0,0 +1,227 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.painless;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.memory.MemoryIndex;
+import org.opensearch.common.collect.Tuple;
+import org.opensearch.common.geo.GeoPoint;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.index.fielddata.IndexGeoPointFieldData;
+import org.opensearch.index.fielddata.IndexNumericFieldData;
+import org.opensearch.index.fielddata.LeafGeoPointFieldData;
+import org.opensearch.index.fielddata.LeafNumericFieldData;
+import org.opensearch.index.fielddata.MultiGeoPointValues;
+import org.opensearch.index.fielddata.SortedNumericDoubleValues;
+import org.opensearch.index.fielddata.plain.AbstractLeafGeoPointFieldData;
+import org.opensearch.index.fielddata.plain.LeafDoubleFieldData;
+import org.opensearch.index.mapper.GeoPointFieldMapper.GeoPointFieldType;
+import org.opensearch.index.mapper.MapperService;
+import org.opensearch.index.mapper.NumberFieldMapper.NumberFieldType;
+import org.opensearch.index.mapper.NumberFieldMapper.NumberType;
+import org.opensearch.painless.spi.Allowlist;
+import org.opensearch.painless.spi.AllowlistLoader;
+import org.opensearch.script.DerivedFieldScript;
+import org.opensearch.script.ScriptContext;
+import org.opensearch.script.ScriptException;
+import org.opensearch.search.lookup.LeafSearchLookup;
+import org.opensearch.search.lookup.SearchLookup;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class DerivedFieldScriptTests extends ScriptTestCase {
+
+    private static PainlessScriptEngine SCRIPT_ENGINE;
+
+    @Override
+    public void setUp() throws Exception {
+        super.setUp();
+
+        // Adding derived field script to the contexts for the script engine
+        Map<ScriptContext<?>, List<Allowlist>> contexts = newDefaultContexts();
+        List<Allowlist> allowlists = new ArrayList<>(Allowlist.BASE_ALLOWLISTS);
+        allowlists.add(AllowlistLoader.loadFromResourceFiles(Allowlist.class, "org.opensearch.derived.txt"));
+        contexts.put(DerivedFieldScript.CONTEXT, allowlists);
+
+        SCRIPT_ENGINE = new PainlessScriptEngine(Settings.EMPTY, contexts);
+    }
+
+    @Override
+    public void tearDown() throws Exception {
+        super.tearDown();
+        SCRIPT_ENGINE = null;
+    }
+
+    @Override
+    protected PainlessScriptEngine getEngine() {
+        return SCRIPT_ENGINE;
+    }
+
+    private DerivedFieldScript.LeafFactory compile(String expression, SearchLookup lookup) {
+        DerivedFieldScript.Factory factory = SCRIPT_ENGINE.compile(
+            "derived_script_test",
+            expression,
+            DerivedFieldScript.CONTEXT,
+            Collections.emptyMap()
+        );
+        return factory.newFactory(Collections.emptyMap(), lookup);
+    }
+
+    public void testEmittingDoubleField() throws IOException {
+        // Mocking field value to be returned
+        NumberFieldType fieldType = new NumberFieldType("test_double_field", NumberType.DOUBLE);
+        MapperService mapperService = mock(MapperService.class);
+        when(mapperService.fieldType("test_double_field")).thenReturn(fieldType);
+
+        SortedNumericDoubleValues doubleValues = mock(SortedNumericDoubleValues.class);
+        when(doubleValues.docValueCount()).thenReturn(1);
+        when(doubleValues.advanceExact(anyInt())).thenReturn(true);
+        when(doubleValues.nextValue()).thenReturn(2.718);
+
+        LeafNumericFieldData atomicFieldData = mock(LeafDoubleFieldData.class); // SortedNumericDoubleFieldData
+        when(atomicFieldData.getDoubleValues()).thenReturn(doubleValues);
+
+        IndexNumericFieldData fieldData = mock(IndexNumericFieldData.class); // SortedNumericIndexFieldData
+        when(fieldData.getFieldName()).thenReturn("test_double_field");
+        when(fieldData.load(any())).thenReturn(atomicFieldData);
+
+        SearchLookup lookup = new SearchLookup(mapperService, (ignored, searchLookup) -> fieldData);
+
+        // We don't need a real index, just need to construct a LeafReaderContext which cannot be mocked
+        MemoryIndex index = new MemoryIndex();
+        LeafReaderContext leafReaderContext = index.createSearcher().getIndexReader().leaves().get(0);
+
+        // Execute the script
+        DerivedFieldScript script = compile("emit(doc['test_double_field'].value)", lookup).newInstance(leafReaderContext);
+        script.setDocument(1);
+        script.execute();
+
+        List<Object> result = script.getEmittedValues();
+        assertEquals(List.of(2.718), result);
+    }
+
+    public void testEmittingGeoPoint() throws IOException {
+        // Mocking field value to be returned
+        GeoPointFieldType fieldType = new GeoPointFieldType("test_geo_field");
+        MapperService mapperService = mock(MapperService.class);
+        when(mapperService.fieldType("test_geo_field")).thenReturn(fieldType);
+
+        MultiGeoPointValues geoPointValues = mock(MultiGeoPointValues.class);
+        when(geoPointValues.docValueCount()).thenReturn(1);
+        when(geoPointValues.advanceExact(anyInt())).thenReturn(true);
+        when(geoPointValues.nextValue()).thenReturn(new GeoPoint(5, 8));
+
+        LeafGeoPointFieldData atomicFieldData = mock(AbstractLeafGeoPointFieldData.class); // LatLonPointDVLeafFieldData
+        when(atomicFieldData.getGeoPointValues()).thenReturn(geoPointValues);
+
+        IndexGeoPointFieldData fieldData = mock(IndexGeoPointFieldData.class);
+        when(fieldData.getFieldName()).thenReturn("test_geo_field");
+        when(fieldData.load(any())).thenReturn(atomicFieldData);
+
+        SearchLookup lookup = new SearchLookup(mapperService, (ignored, searchLookup) -> fieldData);
+
+        // We don't need a real index, just need to construct a LeafReaderContext which cannot be mocked
+        MemoryIndex index = new MemoryIndex();
+        LeafReaderContext leafReaderContext = index.createSearcher().getIndexReader().leaves().get(0);
+
+        // Execute the script
+        DerivedFieldScript script = compile("emit(doc['test_geo_field'].value.getLat(), doc['test_geo_field'].value.getLon())", lookup)
+            .newInstance(leafReaderContext);
+        script.setDocument(1);
+        script.execute();
+
+        List<Object> result = script.getEmittedValues();
+        assertEquals(List.of(new Tuple<>(5.0, 8.0)), result);
+    }
+
+    public void testEmittingMultipleValues() throws IOException {
+        SearchLookup lookup = mock(SearchLookup.class);
+
+        // We don't need a real index, just need to construct a LeafReaderContext which cannot be mocked
+        MemoryIndex index = new MemoryIndex();
+        LeafReaderContext leafReaderContext = index.createSearcher().getIndexReader().leaves().get(0);
+
+        LeafSearchLookup leafSearchLookup = mock(LeafSearchLookup.class);
+        when(lookup.getLeafSearchLookup(leafReaderContext)).thenReturn(leafSearchLookup);
+
+        // Execute the script
+        DerivedFieldScript script = compile(
+            "def l = new ArrayList(); l.add('test'); l.add('multiple'); l.add('values'); for (String x : l) emit(x)",
+            lookup
+        ).newInstance(leafReaderContext);
+        script.setDocument(1);
+        script.execute();
+
+        List<Object> result = script.getEmittedValues();
+        assertEquals(List.of("test", "multiple", "values"), result);
+    }
+
+    public void testExceedingByteSizeLimit() throws IOException {
+        SearchLookup lookup = mock(SearchLookup.class);
+
+        // We don't need a real index, just need to construct a LeafReaderContext which cannot be mocked
+        MemoryIndex index = new MemoryIndex();
+        LeafReaderContext leafReaderContext = index.createSearcher().getIndexReader().leaves().get(0);
+
+        LeafSearchLookup leafSearchLookup = mock(LeafSearchLookup.class);
+        when(lookup.getLeafSearchLookup(leafReaderContext)).thenReturn(leafSearchLookup);
+
+        // Emitting a large string to exceed the byte size limit
+        DerivedFieldScript stringScript = compile("for (int i = 0; i < 1024 * 1024; i++) emit('a' + i);", lookup).newInstance(
+            leafReaderContext
+        );
+        expectThrows(ScriptException.class, () -> {
+            stringScript.setDocument(1);
+            stringScript.execute();
+        });
+
+        // Emitting an integer to check byte size limit
+        DerivedFieldScript intScript = compile("for (int i = 0; i < 1024 * 1024; i++) emit(42)", lookup).newInstance(leafReaderContext);
+        expectThrows(ScriptException.class, "Expected IllegalStateException for exceeding byte size limit", () -> {
+            intScript.setDocument(1);
+            intScript.execute();
+        });
+
+        // Emitting a long to check byte size limit
+        DerivedFieldScript longScript = compile("for (int i = 0; i < 1024 * 1024; i++) emit(1234567890123456789L)", lookup).newInstance(
+            leafReaderContext
+        );
+        expectThrows(ScriptException.class, "Expected IllegalStateException for exceeding byte size limit", () -> {
+            longScript.setDocument(1);
+            longScript.execute();
+        });
+
+        // Emitting a double to check byte size limit
+        DerivedFieldScript doubleScript = compile("for (int i = 0; i < 1024 * 1024; i++) emit(3.14159)", lookup).newInstance(
+            leafReaderContext
+        );
+        expectThrows(ScriptException.class, "Expected IllegalStateException for exceeding byte size limit", () -> {
+            doubleScript.setDocument(1);
+            doubleScript.execute();
+        });
+
+        // Emitting a GeoPoint to check byte size limit
+        DerivedFieldScript geoPointScript = compile("for (int i = 0; i < 1024 * 1024; i++) emit(1.23, 4.56);", lookup).newInstance(
+            leafReaderContext
+        );
+        expectThrows(ScriptException.class, "Expected IllegalStateException for exceeding byte size limit", () -> {
+            geoPointScript.setDocument(1);
+            geoPointScript.execute();
+        });
+    }
+}
diff --git a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/71_context_api.yml b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/71_context_api.yml
index 478ca9ae8abf4..20e6fd351a4b9 100644
--- a/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/71_context_api.yml
+++ b/modules/lang-painless/src/yamlRestTest/resources/rest-api-spec/test/painless/71_context_api.yml
@@ -2,7 +2,7 @@
     - do:
         scripts_painless_context: {}
     - match: { contexts.0: aggregation_selector}
-    - match: { contexts.23: update}
+    - match: { contexts.24: update}
 ---
 
 "Action to get all API values for score context":
diff --git a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapperTests.java b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapperTests.java
index b5f687ce34d4b..f55ad2e9d659c 100644
--- a/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapperTests.java
+++ b/modules/mapper-extras/src/test/java/org/opensearch/index/mapper/SearchAsYouTypeFieldMapperTests.java
@@ -47,6 +47,7 @@
 import org.apache.lucene.search.DisjunctionMaxQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.MultiPhraseQuery;
+import org.apache.lucene.search.NormsFieldExistsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.SynonymQuery;
 import org.apache.lucene.search.TermQuery;
@@ -68,6 +69,7 @@
 import org.opensearch.index.query.MatchPhraseQueryBuilder;
 import org.opensearch.index.query.MultiMatchQueryBuilder;
 import org.opensearch.index.query.QueryShardContext;
+import org.opensearch.index.query.QueryStringQueryBuilder;
 import org.opensearch.plugins.Plugin;
 
 import java.io.IOException;
@@ -541,6 +543,31 @@ public void testMatchPhrase() throws IOException {
         }
     }
 
+    public void testNestedExistsQuery() throws IOException {
+        MapperService mapperService = createMapperService(mapping(b -> {
+            b.startObject("field");
+            {
+                b.field("type", "object");
+                b.startObject("properties");
+                {
+                    b.startObject("nested_field");
+                    {
+                        b.field("type", "search_as_you_type");
+                    }
+                    b.endObject();
+                }
+                b.endObject();
+            }
+            b.endObject();
+        }));
+        QueryShardContext queryShardContext = createQueryShardContext(mapperService);
+        Query actual = new QueryStringQueryBuilder("field:*").toQuery(queryShardContext);
+        Query expected = new ConstantScoreQuery(
+            new BooleanQuery.Builder().add(new NormsFieldExistsQuery("field.nested_field"), BooleanClause.Occur.SHOULD).build()
+        );
+        assertEquals(expected, actual);
+    }
+
     private static BooleanQuery buildBoolPrefixQuery(String shingleFieldName, String prefixFieldName, List<String> terms) {
         final BooleanQuery.Builder builder = new BooleanQuery.Builder();
         for (int i = 0; i < terms.size() - 1; i++) {
diff --git a/modules/rank-eval/src/internalClusterTest/java/org/opensearch/index/rankeval/RankEvalRequestIT.java b/modules/rank-eval/src/internalClusterTest/java/org/opensearch/index/rankeval/RankEvalRequestIT.java
index 488c2e33648e7..0e3db9d1c78b3 100644
--- a/modules/rank-eval/src/internalClusterTest/java/org/opensearch/index/rankeval/RankEvalRequestIT.java
+++ b/modules/rank-eval/src/internalClusterTest/java/org/opensearch/index/rankeval/RankEvalRequestIT.java
@@ -345,7 +345,7 @@ public void testIndicesOptions() {
         request.indicesOptions(IndicesOptions.fromParameters("closed", null, null, "false", SearchRequest.DEFAULT_INDICES_OPTIONS));
         response = client().execute(RankEvalAction.INSTANCE, request).actionGet();
         assertEquals(1, response.getFailures().size());
-        assertThat(response.getFailures().get("amsterdam_query"), instanceOf(IndexClosedException.class));
+        assertThat(response.getFailures().get("amsterdam_query"), instanceOf(IllegalArgumentException.class));
 
         // test allow_no_indices
         request = new RankEvalRequest(task, new String[] { "bad*" });
diff --git a/modules/transport-netty4/licenses/netty-buffer-4.1.107.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-buffer-4.1.107.Final.jar.sha1
deleted file mode 100644
index beb44fc0f4cf9..0000000000000
--- a/modules/transport-netty4/licenses/netty-buffer-4.1.107.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-8509a72b8a5a2d33d611e99254aed39765c3ad82
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-buffer-4.1.109.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-buffer-4.1.109.Final.jar.sha1
new file mode 100644
index 0000000000000..76b51cdae3867
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-buffer-4.1.109.Final.jar.sha1
@@ -0,0 +1 @@
+9d21d602ad7c639fa16b1d26559065d310a34c51
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-codec-4.1.107.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-4.1.107.Final.jar.sha1
deleted file mode 100644
index 4c74bb06fd83b..0000000000000
--- a/modules/transport-netty4/licenses/netty-codec-4.1.107.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-0a1d32debf2ed07c5852ab5b2904c43adb76c39e
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-codec-4.1.109.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-4.1.109.Final.jar.sha1
new file mode 100644
index 0000000000000..1bccee872152d
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-codec-4.1.109.Final.jar.sha1
@@ -0,0 +1 @@
+16e0b2beb49318a549d3ba5d66d707bd5daa8c97
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-codec-http-4.1.107.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http-4.1.107.Final.jar.sha1
deleted file mode 100644
index 38eb2e5bad80a..0000000000000
--- a/modules/transport-netty4/licenses/netty-codec-http-4.1.107.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-04d8e9e51b7254bd26a42fe17bdcae32e4c6ebb3
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-codec-http-4.1.109.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http-4.1.109.Final.jar.sha1
new file mode 100644
index 0000000000000..3423fb94e8497
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-codec-http-4.1.109.Final.jar.sha1
@@ -0,0 +1 @@
+6dca43cedc0b2dc6bf57bdc85fce6ffca3e6b72a
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-codec-http2-4.1.107.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http2-4.1.107.Final.jar.sha1
deleted file mode 100644
index 5b3d3311edc9f..0000000000000
--- a/modules/transport-netty4/licenses/netty-codec-http2-4.1.107.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3885ffe7dd05c9773df70c61009f34a5a8a383ec
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-codec-http2-4.1.109.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-codec-http2-4.1.109.Final.jar.sha1
new file mode 100644
index 0000000000000..b83ad36222d07
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-codec-http2-4.1.109.Final.jar.sha1
@@ -0,0 +1 @@
+6bd4a54b69a81356393f6e4621bad40754f8a5a2
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-common-4.1.107.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-common-4.1.107.Final.jar.sha1
deleted file mode 100644
index bbe91c6ccfb1d..0000000000000
--- a/modules/transport-netty4/licenses/netty-common-4.1.107.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-4f17a547530d64becd7179507b25f4154bcfba57
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-common-4.1.109.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-common-4.1.109.Final.jar.sha1
new file mode 100644
index 0000000000000..5172500557f8b
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-common-4.1.109.Final.jar.sha1
@@ -0,0 +1 @@
+da63e54ee1ca69abf4206cb74fadef7f50850911
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-handler-4.1.107.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-handler-4.1.107.Final.jar.sha1
deleted file mode 100644
index ba27b38632622..0000000000000
--- a/modules/transport-netty4/licenses/netty-handler-4.1.107.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d4c6b05f4d9aca117981297fb7f02953102ebb5e
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-handler-4.1.109.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-handler-4.1.109.Final.jar.sha1
new file mode 100644
index 0000000000000..cabe61b300523
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-handler-4.1.109.Final.jar.sha1
@@ -0,0 +1 @@
+9167863307b3c44cc12262e7b5512de3499b9c4a
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-resolver-4.1.107.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-resolver-4.1.107.Final.jar.sha1
deleted file mode 100644
index 3bc0f7b3fed09..0000000000000
--- a/modules/transport-netty4/licenses/netty-resolver-4.1.107.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-dfee84308341a42131dd0f8ac0e1e02d627c19f3
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-resolver-4.1.109.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-resolver-4.1.109.Final.jar.sha1
new file mode 100644
index 0000000000000..14e21cc0cdb60
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-resolver-4.1.109.Final.jar.sha1
@@ -0,0 +1 @@
+055485ac976e27c8bb67ee111a8490c58f67b70c
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-transport-4.1.107.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-4.1.107.Final.jar.sha1
deleted file mode 100644
index 19419999300dd..0000000000000
--- a/modules/transport-netty4/licenses/netty-transport-4.1.107.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d6a105c621b47d1410e0e09419d7209d2d46e914
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-transport-4.1.109.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-4.1.109.Final.jar.sha1
new file mode 100644
index 0000000000000..6b23d0883e31f
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-transport-4.1.109.Final.jar.sha1
@@ -0,0 +1 @@
+79e3b07d58ef03c7a860d48f932b720675aa8bd3
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.107.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.107.Final.jar.sha1
deleted file mode 100644
index 407ecaffdad30..0000000000000
--- a/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.107.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-4d61d4959741109b3eccd7337f11fc89fa90a74a
\ No newline at end of file
diff --git a/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.109.Final.jar.sha1 b/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.109.Final.jar.sha1
new file mode 100644
index 0000000000000..5afeb9627c9b5
--- /dev/null
+++ b/modules/transport-netty4/licenses/netty-transport-native-unix-common-4.1.109.Final.jar.sha1
@@ -0,0 +1 @@
+da7fe1e6943cbab8ee48df2beadc2c8304f347a2
\ No newline at end of file
diff --git a/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/ssl/SecureNetty4HttpServerTransport.java b/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/ssl/SecureNetty4HttpServerTransport.java
index 51a76903e284d..978c92870bd75 100644
--- a/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/ssl/SecureNetty4HttpServerTransport.java
+++ b/modules/transport-netty4/src/main/java/org/opensearch/http/netty4/ssl/SecureNetty4HttpServerTransport.java
@@ -37,19 +37,27 @@
 import org.opensearch.core.xcontent.NamedXContentRegistry;
 import org.opensearch.http.HttpChannel;
 import org.opensearch.http.HttpHandlingSettings;
+import org.opensearch.http.HttpServerTransport;
 import org.opensearch.http.netty4.Netty4HttpChannel;
 import org.opensearch.http.netty4.Netty4HttpServerTransport;
-import org.opensearch.plugins.SecureTransportSettingsProvider;
+import org.opensearch.plugins.SecureHttpTransportSettingsProvider;
+import org.opensearch.plugins.TransportExceptionHandler;
 import org.opensearch.telemetry.tracing.Tracer;
 import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.transport.SharedGroupFactory;
+import org.opensearch.transport.TransportAdapterProvider;
 import org.opensearch.transport.netty4.ssl.SslUtils;
 
 import javax.net.ssl.SSLEngine;
 
+import java.util.List;
+import java.util.Optional;
+import java.util.stream.Collectors;
+
 import io.netty.channel.Channel;
 import io.netty.channel.ChannelHandler;
 import io.netty.channel.ChannelHandlerContext;
+import io.netty.channel.ChannelInboundHandlerAdapter;
 import io.netty.handler.codec.DecoderException;
 import io.netty.handler.ssl.ApplicationProtocolNames;
 import io.netty.handler.ssl.ApplicationProtocolNegotiationHandler;
@@ -59,9 +67,14 @@
  * @see <a href="https://github.com/opensearch-project/security/blob/d526c9f6c2a438c14db8b413148204510b9fe2e2/src/main/java/org/opensearch/security/ssl/http/netty/SecuritySSLNettyHttpServerTransport.java">SecuritySSLNettyHttpServerTransport</a>
  */
 public class SecureNetty4HttpServerTransport extends Netty4HttpServerTransport {
+    public static final String REQUEST_HEADER_VERIFIER = "HeaderVerifier";
+    public static final String REQUEST_DECOMPRESSOR = "RequestDecompressor";
+
     private static final Logger logger = LogManager.getLogger(SecureNetty4HttpServerTransport.class);
-    private final SecureTransportSettingsProvider secureTransportSettingsProvider;
-    private final SecureTransportSettingsProvider.ServerExceptionHandler exceptionHandler;
+    private final SecureHttpTransportSettingsProvider secureHttpTransportSettingsProvider;
+    private final TransportExceptionHandler exceptionHandler;
+    private final ChannelInboundHandlerAdapter headerVerifier;
+    private final TransportAdapterProvider<HttpServerTransport> decompressorProvider;
 
     public SecureNetty4HttpServerTransport(
         final Settings settings,
@@ -72,7 +85,7 @@ public SecureNetty4HttpServerTransport(
         final Dispatcher dispatcher,
         final ClusterSettings clusterSettings,
         final SharedGroupFactory sharedGroupFactory,
-        final SecureTransportSettingsProvider secureTransportSettingsProvider,
+        final SecureHttpTransportSettingsProvider secureHttpTransportSettingsProvider,
         final Tracer tracer
     ) {
         super(
@@ -86,9 +99,45 @@ public SecureNetty4HttpServerTransport(
             sharedGroupFactory,
             tracer
         );
-        this.secureTransportSettingsProvider = secureTransportSettingsProvider;
-        this.exceptionHandler = secureTransportSettingsProvider.buildHttpServerExceptionHandler(settings, this)
-            .orElse(SecureTransportSettingsProvider.ServerExceptionHandler.NOOP);
+
+        this.secureHttpTransportSettingsProvider = secureHttpTransportSettingsProvider;
+        this.exceptionHandler = secureHttpTransportSettingsProvider.buildHttpServerExceptionHandler(settings, this)
+            .orElse(TransportExceptionHandler.NOOP);
+
+        final List<ChannelInboundHandlerAdapter> headerVerifiers = secureHttpTransportSettingsProvider.getHttpTransportAdapterProviders(
+            settings
+        )
+            .stream()
+            .filter(p -> REQUEST_HEADER_VERIFIER.equalsIgnoreCase(p.name()))
+            .map(p -> p.create(settings, this, ChannelInboundHandlerAdapter.class))
+            .filter(Optional::isPresent)
+            .map(Optional::get)
+            .collect(Collectors.toList());
+
+        if (headerVerifiers.size() > 1) {
+            throw new IllegalArgumentException("Cannot have more than one header verifier configured, supplied " + headerVerifiers.size());
+        }
+
+        final Optional<TransportAdapterProvider<HttpServerTransport>> decompressorProviderOpt = secureHttpTransportSettingsProvider
+            .getHttpTransportAdapterProviders(settings)
+            .stream()
+            .filter(p -> REQUEST_DECOMPRESSOR.equalsIgnoreCase(p.name()))
+            .findFirst();
+        // There could be multiple request decompressor providers configured, using the first one
+        decompressorProviderOpt.ifPresent(p -> logger.debug("Using request decompressor provider: {}", p));
+
+        this.headerVerifier = headerVerifiers.isEmpty() ? null : headerVerifiers.get(0);
+        this.decompressorProvider = decompressorProviderOpt.orElseGet(() -> new TransportAdapterProvider<HttpServerTransport>() {
+            @Override
+            public String name() {
+                return REQUEST_DECOMPRESSOR;
+            }
+
+            @Override
+            public <C> Optional<C> create(Settings settings, HttpServerTransport transport, Class<C> adapterClass) {
+                return Optional.empty();
+            }
+        });
     }
 
     @Override
@@ -152,7 +201,7 @@ protected SslHttpChannelHandler(final Netty4HttpServerTransport transport, final
         protected void initChannel(Channel ch) throws Exception {
             super.initChannel(ch);
 
-            final SSLEngine sslEngine = secureTransportSettingsProvider.buildSecureHttpServerEngine(
+            final SSLEngine sslEngine = secureHttpTransportSettingsProvider.buildSecureHttpServerEngine(
                 settings,
                 SecureNetty4HttpServerTransport.this
             ).orElseGet(SslUtils::createDefaultServerSSLEngine);
@@ -166,4 +215,17 @@ protected void configurePipeline(Channel ch) {
             ch.pipeline().addLast(new Http2OrHttpHandler());
         }
     }
+
+    protected ChannelInboundHandlerAdapter createHeaderVerifier() {
+        if (headerVerifier != null) {
+            return headerVerifier;
+        } else {
+            return super.createHeaderVerifier();
+        }
+    }
+
+    @Override
+    protected ChannelInboundHandlerAdapter createDecompressor() {
+        return decompressorProvider.create(settings, this, ChannelInboundHandlerAdapter.class).orElseGet(super::createDecompressor);
+    }
 }
diff --git a/modules/transport-netty4/src/main/java/org/opensearch/transport/Netty4ModulePlugin.java b/modules/transport-netty4/src/main/java/org/opensearch/transport/Netty4ModulePlugin.java
index 56163c18949a4..e2c84ab5d339a 100644
--- a/modules/transport-netty4/src/main/java/org/opensearch/transport/Netty4ModulePlugin.java
+++ b/modules/transport-netty4/src/main/java/org/opensearch/transport/Netty4ModulePlugin.java
@@ -49,6 +49,7 @@
 import org.opensearch.http.netty4.ssl.SecureNetty4HttpServerTransport;
 import org.opensearch.plugins.NetworkPlugin;
 import org.opensearch.plugins.Plugin;
+import org.opensearch.plugins.SecureHttpTransportSettingsProvider;
 import org.opensearch.plugins.SecureTransportSettingsProvider;
 import org.opensearch.telemetry.tracing.Tracer;
 import org.opensearch.threadpool.ThreadPool;
@@ -160,7 +161,7 @@ public Map<String, Supplier<HttpServerTransport>> getSecureHttpTransports(
         NetworkService networkService,
         HttpServerTransport.Dispatcher dispatcher,
         ClusterSettings clusterSettings,
-        SecureTransportSettingsProvider secureTransportSettingsProvider,
+        SecureHttpTransportSettingsProvider secureHttpTransportSettingsProvider,
         Tracer tracer
     ) {
         return Collections.singletonMap(
@@ -174,7 +175,7 @@ public Map<String, Supplier<HttpServerTransport>> getSecureHttpTransports(
                 dispatcher,
                 clusterSettings,
                 getSharedGroupFactory(settings),
-                secureTransportSettingsProvider,
+                secureHttpTransportSettingsProvider,
                 tracer
             )
         );
diff --git a/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/ssl/SecureNetty4Transport.java b/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/ssl/SecureNetty4Transport.java
index 9c63a1ab9161b..977121346dcc3 100644
--- a/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/ssl/SecureNetty4Transport.java
+++ b/modules/transport-netty4/src/main/java/org/opensearch/transport/netty4/ssl/SecureNetty4Transport.java
@@ -42,6 +42,7 @@
 import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
 import org.opensearch.core.indices.breaker.CircuitBreakerService;
 import org.opensearch.plugins.SecureTransportSettingsProvider;
+import org.opensearch.plugins.TransportExceptionHandler;
 import org.opensearch.telemetry.tracing.Tracer;
 import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.transport.SharedGroupFactory;
@@ -72,7 +73,7 @@ public class SecureNetty4Transport extends Netty4Transport {
 
     private static final Logger logger = LogManager.getLogger(SecureNetty4Transport.class);
     private final SecureTransportSettingsProvider secureTransportSettingsProvider;
-    private final SecureTransportSettingsProvider.ServerExceptionHandler exceptionHandler;
+    private final TransportExceptionHandler exceptionHandler;
 
     public SecureNetty4Transport(
         final Settings settings,
@@ -100,7 +101,7 @@ public SecureNetty4Transport(
 
         this.secureTransportSettingsProvider = secureTransportSettingsProvider;
         this.exceptionHandler = secureTransportSettingsProvider.buildServerTransportExceptionHandler(settings, this)
-            .orElse(SecureTransportSettingsProvider.ServerExceptionHandler.NOOP);
+            .orElse(TransportExceptionHandler.NOOP);
     }
 
     @Override
diff --git a/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpClient.java b/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpClient.java
index 7cc1a47a5d2a4..ef6b67ea44299 100644
--- a/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpClient.java
+++ b/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/Netty4HttpClient.java
@@ -38,7 +38,6 @@
 import org.opensearch.core.common.unit.ByteSizeValue;
 import org.opensearch.tasks.Task;
 import org.opensearch.transport.NettyAllocator;
-import org.opensearch.transport.netty4.ssl.TrustAllManager;
 
 import java.io.Closeable;
 import java.net.SocketAddress;
@@ -90,6 +89,7 @@
 import io.netty.handler.ssl.ClientAuth;
 import io.netty.handler.ssl.SslContextBuilder;
 import io.netty.handler.ssl.SslHandler;
+import io.netty.handler.ssl.util.InsecureTrustManagerFactory;
 import io.netty.util.AttributeKey;
 
 import static io.netty.handler.codec.http.HttpHeaderNames.HOST;
@@ -270,7 +270,7 @@ protected void initChannel(SocketChannel ch) throws Exception {
                 final SslHandler sslHandler = new SslHandler(
                     SslContextBuilder.forClient()
                         .clientAuth(ClientAuth.NONE)
-                        .trustManager(TrustAllManager.INSTANCE)
+                        .trustManager(InsecureTrustManagerFactory.INSTANCE)
                         .build()
                         .newEngine(ch.alloc())
                 );
diff --git a/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/ssl/SecureNetty4HttpServerTransportConfigurationTests.java b/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/ssl/SecureNetty4HttpServerTransportConfigurationTests.java
new file mode 100644
index 0000000000000..1ab1ae4f5ddfd
--- /dev/null
+++ b/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/ssl/SecureNetty4HttpServerTransportConfigurationTests.java
@@ -0,0 +1,242 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.http.netty4.ssl;
+
+import org.opensearch.common.network.NetworkService;
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.MockBigArrays;
+import org.opensearch.common.util.MockPageCacheRecycler;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
+import org.opensearch.http.HttpServerTransport;
+import org.opensearch.http.NullDispatcher;
+import org.opensearch.plugins.SecureHttpTransportSettingsProvider;
+import org.opensearch.plugins.TransportExceptionHandler;
+import org.opensearch.telemetry.tracing.noop.NoopTracer;
+import org.opensearch.test.OpenSearchTestCase;
+import org.opensearch.threadpool.TestThreadPool;
+import org.opensearch.threadpool.ThreadPool;
+import org.opensearch.transport.SharedGroupFactory;
+import org.opensearch.transport.TransportAdapterProvider;
+import org.junit.After;
+import org.junit.Before;
+
+import javax.net.ssl.SSLEngine;
+import javax.net.ssl.SSLException;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Optional;
+
+import io.netty.channel.ChannelInboundHandlerAdapter;
+
+import static org.hamcrest.Matchers.equalTo;
+
+/**
+ * Tests for the {@link SecureNetty4HttpServerTransport} class.
+ */
+public class SecureNetty4HttpServerTransportConfigurationTests extends OpenSearchTestCase {
+
+    private NetworkService networkService;
+    private ThreadPool threadPool;
+    private MockBigArrays bigArrays;
+    private ClusterSettings clusterSettings;
+
+    private static class ConfigurableSecureHttpTransportSettingsProvider implements SecureHttpTransportSettingsProvider {
+        private final List<TransportAdapterProvider<HttpServerTransport>> transportAdapterProviders;
+
+        public ConfigurableSecureHttpTransportSettingsProvider(
+            List<TransportAdapterProvider<HttpServerTransport>> transportAdapterProviders
+        ) {
+            this.transportAdapterProviders = transportAdapterProviders;
+        }
+
+        @Override
+        public Collection<TransportAdapterProvider<HttpServerTransport>> getHttpTransportAdapterProviders(Settings settings) {
+            return transportAdapterProviders;
+        }
+
+        @Override
+        public Optional<TransportExceptionHandler> buildHttpServerExceptionHandler(Settings settings, HttpServerTransport transport) {
+            return Optional.empty();
+        }
+
+        @Override
+        public Optional<SSLEngine> buildSecureHttpServerEngine(Settings settings, HttpServerTransport transport) throws SSLException {
+            return Optional.empty();
+        }
+    }
+
+    @Before
+    public void setup() throws Exception {
+        networkService = new NetworkService(Collections.emptyList());
+        threadPool = new TestThreadPool("test");
+        bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService());
+        clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+    }
+
+    @After
+    public void shutdown() throws Exception {
+        if (threadPool != null) {
+            threadPool.shutdownNow();
+        }
+        threadPool = null;
+        networkService = null;
+        bigArrays = null;
+        clusterSettings = null;
+    }
+
+    public void testRequestHeaderVerifier() throws InterruptedException {
+        final TransportAdapterProvider<HttpServerTransport> transportAdapterProvider = new TransportAdapterProvider<HttpServerTransport>() {
+            @Override
+            public String name() {
+                return SecureNetty4HttpServerTransport.REQUEST_HEADER_VERIFIER;
+            }
+
+            @SuppressWarnings("unchecked")
+            @Override
+            public <C> Optional<C> create(Settings settings, HttpServerTransport transport, Class<C> adapterClass) {
+                return Optional.of((C) new ChannelInboundHandlerAdapter());
+            }
+
+        };
+
+        try (
+            final SecureNetty4HttpServerTransport transport = new SecureNetty4HttpServerTransport(
+                Settings.EMPTY,
+                networkService,
+                bigArrays,
+                threadPool,
+                xContentRegistry(),
+                new NullDispatcher(),
+                clusterSettings,
+                new SharedGroupFactory(Settings.EMPTY),
+                new ConfigurableSecureHttpTransportSettingsProvider(List.of(transportAdapterProvider)),
+                NoopTracer.INSTANCE
+            )
+        ) {
+
+        }
+    }
+
+    public void testMultipleRequestHeaderVerifiers() throws InterruptedException {
+        final TransportAdapterProvider<HttpServerTransport> transportAdapterProvider = new TransportAdapterProvider<HttpServerTransport>() {
+            @Override
+            public String name() {
+                return SecureNetty4HttpServerTransport.REQUEST_HEADER_VERIFIER;
+            }
+
+            @SuppressWarnings("unchecked")
+            @Override
+            public <C> Optional<C> create(Settings settings, HttpServerTransport transport, Class<C> adapterClass) {
+                return Optional.of((C) new ChannelInboundHandlerAdapter());
+            }
+
+        };
+
+        final IllegalArgumentException ex = assertThrows(
+            IllegalArgumentException.class,
+            () -> new SecureNetty4HttpServerTransport(
+                Settings.EMPTY,
+                networkService,
+                bigArrays,
+                threadPool,
+                xContentRegistry(),
+                new NullDispatcher(),
+                clusterSettings,
+                new SharedGroupFactory(Settings.EMPTY),
+                new ConfigurableSecureHttpTransportSettingsProvider(List.of(transportAdapterProvider, transportAdapterProvider)),
+                NoopTracer.INSTANCE
+            )
+        );
+
+        assertThat(ex.getMessage(), equalTo("Cannot have more than one header verifier configured, supplied 2"));
+    }
+
+    public void testRequestDecompressor() throws InterruptedException {
+        final TransportAdapterProvider<HttpServerTransport> transportAdapterProvider = new TransportAdapterProvider<HttpServerTransport>() {
+            @Override
+            public String name() {
+                return SecureNetty4HttpServerTransport.REQUEST_DECOMPRESSOR;
+            }
+
+            @SuppressWarnings("unchecked")
+            @Override
+            public <C> Optional<C> create(Settings settings, HttpServerTransport transport, Class<C> adapterClass) {
+                return Optional.of((C) new ChannelInboundHandlerAdapter());
+            }
+
+        };
+
+        try (
+            final SecureNetty4HttpServerTransport transport = new SecureNetty4HttpServerTransport(
+                Settings.EMPTY,
+                networkService,
+                bigArrays,
+                threadPool,
+                xContentRegistry(),
+                new NullDispatcher(),
+                clusterSettings,
+                new SharedGroupFactory(Settings.EMPTY),
+                new ConfigurableSecureHttpTransportSettingsProvider(List.of(transportAdapterProvider)),
+                NoopTracer.INSTANCE
+            )
+        ) {
+
+        }
+    }
+
+    public void testRequestDecompressorAndRequestHeaderVerifier() throws InterruptedException {
+        final TransportAdapterProvider<HttpServerTransport> requestDecompressor = new TransportAdapterProvider<HttpServerTransport>() {
+            @Override
+            public String name() {
+                return SecureNetty4HttpServerTransport.REQUEST_DECOMPRESSOR;
+            }
+
+            @SuppressWarnings("unchecked")
+            @Override
+            public <C> Optional<C> create(Settings settings, HttpServerTransport transport, Class<C> adapterClass) {
+                return Optional.of((C) new ChannelInboundHandlerAdapter());
+            }
+
+        };
+
+        final TransportAdapterProvider<HttpServerTransport> requestHeaderVerifier = new TransportAdapterProvider<HttpServerTransport>() {
+            @Override
+            public String name() {
+                return SecureNetty4HttpServerTransport.REQUEST_HEADER_VERIFIER;
+            }
+
+            @SuppressWarnings("unchecked")
+            @Override
+            public <C> Optional<C> create(Settings settings, HttpServerTransport transport, Class<C> adapterClass) {
+                return Optional.of((C) new ChannelInboundHandlerAdapter());
+            }
+
+        };
+
+        try (
+            final SecureNetty4HttpServerTransport transport = new SecureNetty4HttpServerTransport(
+                Settings.EMPTY,
+                networkService,
+                bigArrays,
+                threadPool,
+                xContentRegistry(),
+                new NullDispatcher(),
+                clusterSettings,
+                new SharedGroupFactory(Settings.EMPTY),
+                new ConfigurableSecureHttpTransportSettingsProvider(List.of(requestDecompressor, requestHeaderVerifier)),
+                NoopTracer.INSTANCE
+            )
+        ) {
+
+        }
+    }
+}
diff --git a/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/ssl/SecureNetty4HttpServerTransportTests.java b/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/ssl/SecureNetty4HttpServerTransportTests.java
index 9ea49d0b24d44..f80ad901ce765 100644
--- a/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/ssl/SecureNetty4HttpServerTransportTests.java
+++ b/modules/transport-netty4/src/test/java/org/opensearch/http/netty4/ssl/SecureNetty4HttpServerTransportTests.java
@@ -29,7 +29,8 @@
 import org.opensearch.http.HttpTransportSettings;
 import org.opensearch.http.NullDispatcher;
 import org.opensearch.http.netty4.Netty4HttpClient;
-import org.opensearch.plugins.SecureTransportSettingsProvider;
+import org.opensearch.plugins.SecureHttpTransportSettingsProvider;
+import org.opensearch.plugins.TransportExceptionHandler;
 import org.opensearch.rest.BytesRestResponse;
 import org.opensearch.rest.RestChannel;
 import org.opensearch.rest.RestRequest;
@@ -40,8 +41,6 @@
 import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.transport.NettyAllocator;
 import org.opensearch.transport.SharedGroupFactory;
-import org.opensearch.transport.TcpTransport;
-import org.opensearch.transport.netty4.ssl.TrustAllManager;
 import org.junit.After;
 import org.junit.Before;
 
@@ -83,8 +82,8 @@
 import io.netty.handler.codec.http.HttpResponseStatus;
 import io.netty.handler.codec.http.HttpUtil;
 import io.netty.handler.codec.http.HttpVersion;
-import io.netty.handler.ssl.ClientAuth;
 import io.netty.handler.ssl.SslContextBuilder;
+import io.netty.handler.ssl.util.InsecureTrustManagerFactory;
 
 import static org.opensearch.core.rest.RestStatus.BAD_REQUEST;
 import static org.opensearch.core.rest.RestStatus.OK;
@@ -104,7 +103,7 @@ public class SecureNetty4HttpServerTransportTests extends OpenSearchTestCase {
     private ThreadPool threadPool;
     private MockBigArrays bigArrays;
     private ClusterSettings clusterSettings;
-    private SecureTransportSettingsProvider secureTransportSettingsProvider;
+    private SecureHttpTransportSettingsProvider secureHttpTransportSettingsProvider;
 
     @Before
     public void setup() throws Exception {
@@ -113,14 +112,9 @@ public void setup() throws Exception {
         bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService());
         clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
 
-        secureTransportSettingsProvider = new SecureTransportSettingsProvider() {
+        secureHttpTransportSettingsProvider = new SecureHttpTransportSettingsProvider() {
             @Override
-            public Optional<ServerExceptionHandler> buildHttpServerExceptionHandler(Settings settings, HttpServerTransport transport) {
-                return Optional.empty();
-            }
-
-            @Override
-            public Optional<ServerExceptionHandler> buildServerTransportExceptionHandler(Settings settings, TcpTransport transport) {
+            public Optional<TransportExceptionHandler> buildHttpServerExceptionHandler(Settings settings, HttpServerTransport transport) {
                 return Optional.empty();
             }
 
@@ -137,7 +131,7 @@ public Optional<SSLEngine> buildSecureHttpServerEngine(Settings settings, HttpSe
                     keyManagerFactory.init(keyStore, "password".toCharArray());
 
                     SSLEngine engine = SslContextBuilder.forServer(keyManagerFactory)
-                        .trustManager(TrustAllManager.INSTANCE)
+                        .trustManager(InsecureTrustManagerFactory.INSTANCE)
                         .build()
                         .newEngine(NettyAllocator.getAllocator());
                     return Optional.of(engine);
@@ -146,22 +140,6 @@ public Optional<SSLEngine> buildSecureHttpServerEngine(Settings settings, HttpSe
                     throw new SSLException(ex);
                 }
             }
-
-            @Override
-            public Optional<SSLEngine> buildSecureServerTransportEngine(Settings settings, TcpTransport transport) throws SSLException {
-                return Optional.empty();
-            }
-
-            @Override
-            public Optional<SSLEngine> buildSecureClientTransportEngine(Settings settings, String hostname, int port) throws SSLException {
-                return Optional.of(
-                    SslContextBuilder.forClient()
-                        .clientAuth(ClientAuth.NONE)
-                        .trustManager(TrustAllManager.INSTANCE)
-                        .build()
-                        .newEngine(NettyAllocator.getAllocator())
-                );
-            }
         };
     }
 
@@ -241,7 +219,7 @@ public void dispatchBadRequest(RestChannel channel, ThreadContext threadContext,
                 dispatcher,
                 clusterSettings,
                 new SharedGroupFactory(settings),
-                secureTransportSettingsProvider,
+                secureHttpTransportSettingsProvider,
                 NoopTracer.INSTANCE
             )
         ) {
@@ -292,7 +270,7 @@ public void testBindUnavailableAddress() {
                 new NullDispatcher(),
                 clusterSettings,
                 new SharedGroupFactory(Settings.EMPTY),
-                secureTransportSettingsProvider,
+                secureHttpTransportSettingsProvider,
                 NoopTracer.INSTANCE
             )
         ) {
@@ -312,7 +290,7 @@ public void testBindUnavailableAddress() {
                     new NullDispatcher(),
                     clusterSettings,
                     new SharedGroupFactory(settings),
-                    secureTransportSettingsProvider,
+                    secureHttpTransportSettingsProvider,
                     NoopTracer.INSTANCE
                 )
             ) {
@@ -366,7 +344,7 @@ public void dispatchBadRequest(final RestChannel channel, final ThreadContext th
                 dispatcher,
                 clusterSettings,
                 new SharedGroupFactory(settings),
-                secureTransportSettingsProvider,
+                secureHttpTransportSettingsProvider,
                 NoopTracer.INSTANCE
             )
         ) {
@@ -430,7 +408,7 @@ public void dispatchBadRequest(final RestChannel channel, final ThreadContext th
                 dispatcher,
                 clusterSettings,
                 new SharedGroupFactory(Settings.EMPTY),
-                secureTransportSettingsProvider,
+                secureHttpTransportSettingsProvider,
                 NoopTracer.INSTANCE
             )
         ) {
@@ -487,7 +465,7 @@ public void dispatchBadRequest(final RestChannel channel, final ThreadContext th
                 dispatcher,
                 new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
                 new SharedGroupFactory(settings),
-                secureTransportSettingsProvider,
+                secureHttpTransportSettingsProvider,
                 NoopTracer.INSTANCE
             )
         ) {
@@ -562,7 +540,7 @@ public void dispatchBadRequest(final RestChannel channel, final ThreadContext th
                 dispatcher,
                 new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
                 new SharedGroupFactory(settings),
-                secureTransportSettingsProvider,
+                secureHttpTransportSettingsProvider,
                 NoopTracer.INSTANCE
             )
         ) {
diff --git a/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/ssl/SimpleSecureNetty4TransportTests.java b/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/ssl/SimpleSecureNetty4TransportTests.java
index 0cae58b8efa2a..e0600aebd90e5 100644
--- a/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/ssl/SimpleSecureNetty4TransportTests.java
+++ b/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/ssl/SimpleSecureNetty4TransportTests.java
@@ -20,8 +20,8 @@
 import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
 import org.opensearch.core.common.transport.TransportAddress;
 import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
-import org.opensearch.http.HttpServerTransport;
 import org.opensearch.plugins.SecureTransportSettingsProvider;
+import org.opensearch.plugins.TransportExceptionHandler;
 import org.opensearch.telemetry.tracing.noop.NoopTracer;
 import org.opensearch.test.transport.MockTransportService;
 import org.opensearch.test.transport.StubbableTransport;
@@ -55,6 +55,7 @@
 
 import io.netty.handler.ssl.ClientAuth;
 import io.netty.handler.ssl.SslContextBuilder;
+import io.netty.handler.ssl.util.InsecureTrustManagerFactory;
 
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
@@ -69,40 +70,12 @@ protected Transport build(Settings settings, final Version version, ClusterSetti
         NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(Collections.emptyList());
         final SecureTransportSettingsProvider secureTransportSettingsProvider = new SecureTransportSettingsProvider() {
             @Override
-            public Optional<ServerExceptionHandler> buildHttpServerExceptionHandler(Settings settings, HttpServerTransport transport) {
+            public Optional<TransportExceptionHandler> buildServerTransportExceptionHandler(Settings settings, Transport transport) {
                 return Optional.empty();
             }
 
             @Override
-            public Optional<ServerExceptionHandler> buildServerTransportExceptionHandler(Settings settings, TcpTransport transport) {
-                return Optional.empty();
-            }
-
-            @Override
-            public Optional<SSLEngine> buildSecureHttpServerEngine(Settings settings, HttpServerTransport transport) throws SSLException {
-                try {
-                    final KeyStore keyStore = KeyStore.getInstance("PKCS12");
-                    keyStore.load(
-                        SimpleSecureNetty4TransportTests.class.getResourceAsStream("/netty4-secure.jks"),
-                        "password".toCharArray()
-                    );
-
-                    final KeyManagerFactory keyManagerFactory = KeyManagerFactory.getInstance("SunX509");
-                    keyManagerFactory.init(keyStore, "password".toCharArray());
-
-                    SSLEngine engine = SslContextBuilder.forServer(keyManagerFactory)
-                        .trustManager(TrustAllManager.INSTANCE)
-                        .build()
-                        .newEngine(NettyAllocator.getAllocator());
-                    return Optional.of(engine);
-                } catch (final IOException | NoSuchAlgorithmException | UnrecoverableKeyException | KeyStoreException
-                    | CertificateException ex) {
-                    throw new SSLException(ex);
-                }
-            }
-
-            @Override
-            public Optional<SSLEngine> buildSecureServerTransportEngine(Settings settings, TcpTransport transport) throws SSLException {
+            public Optional<SSLEngine> buildSecureServerTransportEngine(Settings settings, Transport transport) throws SSLException {
                 try {
                     final KeyStore keyStore = KeyStore.getInstance("PKCS12");
                     keyStore.load(
@@ -115,7 +88,7 @@ public Optional<SSLEngine> buildSecureServerTransportEngine(Settings settings, T
 
                     SSLEngine engine = SslContextBuilder.forServer(keyManagerFactory)
                         .clientAuth(ClientAuth.NONE)
-                        .trustManager(TrustAllManager.INSTANCE)
+                        .trustManager(InsecureTrustManagerFactory.INSTANCE)
                         .build()
                         .newEngine(NettyAllocator.getAllocator());
                     return Optional.of(engine);
@@ -131,7 +104,7 @@ public Optional<SSLEngine> buildSecureClientTransportEngine(Settings settings, S
                 return Optional.of(
                     SslContextBuilder.forClient()
                         .clientAuth(ClientAuth.NONE)
-                        .trustManager(TrustAllManager.INSTANCE)
+                        .trustManager(InsecureTrustManagerFactory.INSTANCE)
                         .build()
                         .newEngine(NettyAllocator.getAllocator())
                 );
diff --git a/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/ssl/TrustAllManager.java b/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/ssl/TrustAllManager.java
deleted file mode 100644
index a38c542b5780e..0000000000000
--- a/modules/transport-netty4/src/test/java/org/opensearch/transport/netty4/ssl/TrustAllManager.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.transport.netty4.ssl;
-
-import javax.net.ssl.X509TrustManager;
-
-import java.security.cert.CertificateException;
-import java.security.cert.X509Certificate;
-
-public class TrustAllManager implements X509TrustManager {
-    public static final X509TrustManager INSTANCE = new TrustAllManager();
-
-    private TrustAllManager() {}
-
-    public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException {}
-
-    public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException {}
-
-    public X509Certificate[] getAcceptedIssuers() {
-        return new X509Certificate[0];
-    }
-}
diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.11.0-snapshot-8a555eb.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.11.0-snapshot-8a555eb.jar.sha1
deleted file mode 100644
index 0643f16dc1052..0000000000000
--- a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.11.0-snapshot-8a555eb.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-44a4e095d7e047a9452d81b224905b72c830f8ae
\ No newline at end of file
diff --git a/plugins/analysis-icu/licenses/lucene-analysis-icu-9.11.0-snapshot-fb97840.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.11.0-snapshot-fb97840.jar.sha1
new file mode 100644
index 0000000000000..594733c11402c
--- /dev/null
+++ b/plugins/analysis-icu/licenses/lucene-analysis-icu-9.11.0-snapshot-fb97840.jar.sha1
@@ -0,0 +1 @@
+f9cd7bec33c8cf3b891976cb674ffe9c97f8747f
\ No newline at end of file
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.11.0-snapshot-8a555eb.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.11.0-snapshot-8a555eb.jar.sha1
deleted file mode 100644
index 3f4d49a78791b..0000000000000
--- a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.11.0-snapshot-8a555eb.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-7f57fe7322e6d3a9b4edcc3da0b1ee0791a814ec
\ No newline at end of file
diff --git a/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.11.0-snapshot-fb97840.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.11.0-snapshot-fb97840.jar.sha1
new file mode 100644
index 0000000000000..c46e06b8c87e4
--- /dev/null
+++ b/plugins/analysis-kuromoji/licenses/lucene-analysis-kuromoji-9.11.0-snapshot-fb97840.jar.sha1
@@ -0,0 +1 @@
+c244a56bf7cd171a19379c96f1d20c477a34578d
\ No newline at end of file
diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.11.0-snapshot-8a555eb.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.11.0-snapshot-8a555eb.jar.sha1
deleted file mode 100644
index 1f110011ca9c6..0000000000000
--- a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.11.0-snapshot-8a555eb.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-9929da235100f8df323cfed165b8111fb2840093
\ No newline at end of file
diff --git a/plugins/analysis-nori/licenses/lucene-analysis-nori-9.11.0-snapshot-fb97840.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.11.0-snapshot-fb97840.jar.sha1
new file mode 100644
index 0000000000000..a79c34a127920
--- /dev/null
+++ b/plugins/analysis-nori/licenses/lucene-analysis-nori-9.11.0-snapshot-fb97840.jar.sha1
@@ -0,0 +1 @@
+da26df43f2b0d7c2dfecbf208cae0772a5e382ca
\ No newline at end of file
diff --git a/plugins/analysis-phonetic/licenses/commons-codec-1.15.jar.sha1 b/plugins/analysis-phonetic/licenses/commons-codec-1.15.jar.sha1
deleted file mode 100644
index 62d99837b87e1..0000000000000
--- a/plugins/analysis-phonetic/licenses/commons-codec-1.15.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-49d94806b6e3dc933dacbd8acb0fdbab8ebd1e5d
\ No newline at end of file
diff --git a/plugins/analysis-phonetic/licenses/commons-codec-1.16.1.jar.sha1 b/plugins/analysis-phonetic/licenses/commons-codec-1.16.1.jar.sha1
new file mode 100644
index 0000000000000..6b8803089c6d7
--- /dev/null
+++ b/plugins/analysis-phonetic/licenses/commons-codec-1.16.1.jar.sha1
@@ -0,0 +1 @@
+47bd4d333fba53406f6c6c51884ddbca435c8862
\ No newline at end of file
diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.11.0-snapshot-8a555eb.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.11.0-snapshot-8a555eb.jar.sha1
deleted file mode 100644
index 8d6bf9fa0fa1b..0000000000000
--- a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.11.0-snapshot-8a555eb.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-8564c86d880c6ce002250002e2fd0936cbfff61d
\ No newline at end of file
diff --git a/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.11.0-snapshot-fb97840.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.11.0-snapshot-fb97840.jar.sha1
new file mode 100644
index 0000000000000..f2b08020be1ad
--- /dev/null
+++ b/plugins/analysis-phonetic/licenses/lucene-analysis-phonetic-9.11.0-snapshot-fb97840.jar.sha1
@@ -0,0 +1 @@
+f752ffa5ee4697b04643214236138f3defdee2f4
\ No newline at end of file
diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.11.0-snapshot-8a555eb.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.11.0-snapshot-8a555eb.jar.sha1
deleted file mode 100644
index cbe4aec98fae4..0000000000000
--- a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.11.0-snapshot-8a555eb.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-7e71777cfb5beb4ffd5b03030576d2f062eef13c
\ No newline at end of file
diff --git a/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.11.0-snapshot-fb97840.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.11.0-snapshot-fb97840.jar.sha1
new file mode 100644
index 0000000000000..969a05905eaf0
--- /dev/null
+++ b/plugins/analysis-smartcn/licenses/lucene-analysis-smartcn-9.11.0-snapshot-fb97840.jar.sha1
@@ -0,0 +1 @@
+73fe44fe755aef72e7293b2ffdd934beb631429d
\ No newline at end of file
diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.11.0-snapshot-8a555eb.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.11.0-snapshot-8a555eb.jar.sha1
deleted file mode 100644
index e1c7aecc104d0..0000000000000
--- a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.11.0-snapshot-8a555eb.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-a597265bd6fb0a7e954e948a295d31507dd73cce
\ No newline at end of file
diff --git a/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.11.0-snapshot-fb97840.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.11.0-snapshot-fb97840.jar.sha1
new file mode 100644
index 0000000000000..fdf0bd39e217e
--- /dev/null
+++ b/plugins/analysis-stempel/licenses/lucene-analysis-stempel-9.11.0-snapshot-fb97840.jar.sha1
@@ -0,0 +1 @@
+c2b48831b25e1c7e8f683a63d1505c2d133256d3
\ No newline at end of file
diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.11.0-snapshot-8a555eb.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.11.0-snapshot-8a555eb.jar.sha1
deleted file mode 100644
index eefa2809f3540..0000000000000
--- a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.11.0-snapshot-8a555eb.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-c9e534845bb08985d7fa21e2e71a14bc68c46089
\ No newline at end of file
diff --git a/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.11.0-snapshot-fb97840.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.11.0-snapshot-fb97840.jar.sha1
new file mode 100644
index 0000000000000..0042415700453
--- /dev/null
+++ b/plugins/analysis-ukrainian/licenses/lucene-analysis-morfologik-9.11.0-snapshot-fb97840.jar.sha1
@@ -0,0 +1 @@
+757f8b29f103f82b6fb6948634e93dd497c9d7a8
\ No newline at end of file
diff --git a/plugins/cache-ehcache/build.gradle b/plugins/cache-ehcache/build.gradle
index 65e7daaaacf26..4fc5e44f58c3a 100644
--- a/plugins/cache-ehcache/build.gradle
+++ b/plugins/cache-ehcache/build.gradle
@@ -95,3 +95,8 @@ test {
   // TODO: Adding permission in plugin-security.policy doesn't seem to work.
   systemProperty 'tests.security.manager', 'false'
 }
+
+internalClusterTest {
+  // TODO: Remove this later once we have a way.
+  systemProperty 'tests.security.manager', 'false'
+}
diff --git a/plugins/cache-ehcache/src/internalClusterTest/java/org/opensearch/cache/EhcacheDiskCacheIT.java b/plugins/cache-ehcache/src/internalClusterTest/java/org/opensearch/cache/EhcacheDiskCacheIT.java
index c68455463ee3d..909a493c0734f 100644
--- a/plugins/cache-ehcache/src/internalClusterTest/java/org/opensearch/cache/EhcacheDiskCacheIT.java
+++ b/plugins/cache-ehcache/src/internalClusterTest/java/org/opensearch/cache/EhcacheDiskCacheIT.java
@@ -8,22 +8,62 @@
 
 package org.opensearch.cache;
 
+import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
+
 import org.opensearch.action.admin.cluster.node.info.NodeInfo;
 import org.opensearch.action.admin.cluster.node.info.NodesInfoRequest;
 import org.opensearch.action.admin.cluster.node.info.NodesInfoResponse;
 import org.opensearch.action.admin.cluster.node.info.PluginsAndModules;
+import org.opensearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest;
+import org.opensearch.action.admin.indices.cache.clear.ClearIndicesCacheResponse;
+import org.opensearch.action.admin.indices.forcemerge.ForceMergeResponse;
+import org.opensearch.action.search.SearchResponse;
+import org.opensearch.action.search.SearchType;
+import org.opensearch.cache.store.disk.EhcacheDiskCache;
+import org.opensearch.cache.store.disk.EhcacheThreadLeakFilter;
+import org.opensearch.client.Client;
+import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.common.cache.CacheType;
+import org.opensearch.common.cache.settings.CacheSettings;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.TimeValue;
+import org.opensearch.common.util.FeatureFlags;
+import org.opensearch.env.NodeEnvironment;
+import org.opensearch.index.cache.request.RequestCacheStats;
+import org.opensearch.index.query.QueryBuilders;
+import org.opensearch.indices.IndicesRequestCache;
 import org.opensearch.plugins.Plugin;
 import org.opensearch.plugins.PluginInfo;
+import org.opensearch.search.aggregations.bucket.histogram.DateHistogramInterval;
 import org.opensearch.test.OpenSearchIntegTestCase;
+import org.opensearch.test.hamcrest.OpenSearchAssertions;
 import org.junit.Assert;
 
+import java.io.IOException;
+import java.time.ZoneId;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+import static org.opensearch.cache.EhcacheDiskCacheSettings.DEFAULT_CACHE_SIZE_IN_BYTES;
+import static org.opensearch.cache.EhcacheDiskCacheSettings.DISK_CACHE_EXPIRE_AFTER_ACCESS_KEY;
+import static org.opensearch.cache.EhcacheDiskCacheSettings.DISK_LISTENER_MODE_SYNC_KEY;
+import static org.opensearch.cache.EhcacheDiskCacheSettings.DISK_MAX_SIZE_IN_BYTES_KEY;
+import static org.opensearch.cache.EhcacheDiskCacheSettings.DISK_STORAGE_PATH_KEY;
+import static org.opensearch.indices.IndicesService.INDICES_CACHE_CLEAN_INTERVAL_SETTING;
+import static org.opensearch.search.aggregations.AggregationBuilders.dateHistogram;
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertNoFailures;
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse;
+import static org.hamcrest.Matchers.greaterThan;
+
+@OpenSearchIntegTestCase.ClusterScope(numDataNodes = 0, scope = OpenSearchIntegTestCase.Scope.TEST)
+@ThreadLeakFilters(filters = { EhcacheThreadLeakFilter.class })
 public class EhcacheDiskCacheIT extends OpenSearchIntegTestCase {
 
     @Override
@@ -31,7 +71,53 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
         return Arrays.asList(EhcacheCachePlugin.class);
     }
 
+    @Override
+    protected Settings featureFlagSettings() {
+        return Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.PLUGGABLE_CACHE, "true").build();
+    }
+
+    private Settings defaultSettings(long sizeInBytes, TimeValue expirationTime) {
+        if (expirationTime == null) {
+            expirationTime = TimeValue.MAX_VALUE;
+        }
+        try (NodeEnvironment env = newNodeEnvironment(Settings.EMPTY)) {
+            return Settings.builder()
+                .put(
+                    EhcacheDiskCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE)
+                        .get(DISK_STORAGE_PATH_KEY)
+                        .getKey(),
+                    env.nodePaths()[0].indicesPath.toString() + "/" + UUID.randomUUID() + "/request_cache/"
+                )
+                .put(
+                    CacheSettings.getConcreteStoreNameSettingForCacheType(CacheType.INDICES_REQUEST_CACHE).getKey(),
+                    EhcacheDiskCache.EhcacheDiskCacheFactory.EHCACHE_DISK_CACHE_NAME
+                )
+                .put(
+                    EhcacheDiskCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE)
+                        .get(DISK_LISTENER_MODE_SYNC_KEY)
+                        .getKey(),
+                    true
+                )
+                .put(
+                    EhcacheDiskCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE)
+                        .get(DISK_MAX_SIZE_IN_BYTES_KEY)
+                        .getKey(),
+                    sizeInBytes
+                )
+                .put(
+                    EhcacheDiskCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE)
+                        .get(DISK_CACHE_EXPIRE_AFTER_ACCESS_KEY)
+                        .getKey(),
+                    expirationTime
+                )
+                .build();
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+
     public void testPluginsAreInstalled() {
+        internalCluster().startNode(Settings.builder().put(defaultSettings(DEFAULT_CACHE_SIZE_IN_BYTES, null)).build());
         NodesInfoRequest nodesInfoRequest = new NodesInfoRequest();
         nodesInfoRequest.addMetric(NodesInfoRequest.Metric.PLUGINS.metricName());
         NodesInfoResponse nodesInfoResponse = OpenSearchIntegTestCase.client().admin().cluster().nodesInfo(nodesInfoRequest).actionGet();
@@ -45,4 +131,258 @@ public void testPluginsAreInstalled() {
             pluginInfos.stream().anyMatch(pluginInfo -> pluginInfo.getName().equals("org.opensearch.cache.EhcacheCachePlugin"))
         );
     }
+
+    public void testSanityChecksWithIndicesRequestCache() throws InterruptedException {
+        internalCluster().startNode(Settings.builder().put(defaultSettings(DEFAULT_CACHE_SIZE_IN_BYTES, null)).build());
+        Client client = client();
+        assertAcked(
+            client.admin()
+                .indices()
+                .prepareCreate("index")
+                .setMapping("f", "type=date")
+                .setSettings(
+                    Settings.builder()
+                        .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                        .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                        .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
+                        .build()
+                )
+                .get()
+        );
+        indexRandom(
+            true,
+            client.prepareIndex("index").setSource("f", "2014-03-10T00:00:00.000Z"),
+            client.prepareIndex("index").setSource("f", "2014-05-13T00:00:00.000Z")
+        );
+        ensureSearchable("index");
+
+        // This is not a random example: serialization with time zones writes shared strings
+        // which used to not work well with the query cache because of the handles stream output
+        // see #9500
+        final SearchResponse r1 = client.prepareSearch("index")
+            .setSize(0)
+            .setSearchType(SearchType.QUERY_THEN_FETCH)
+            .addAggregation(
+                dateHistogram("histo").field("f")
+                    .timeZone(ZoneId.of("+01:00"))
+                    .minDocCount(0)
+                    .dateHistogramInterval(DateHistogramInterval.MONTH)
+            )
+            .get();
+        assertSearchResponse(r1);
+
+        // The cached is actually used
+        assertThat(
+            client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMemorySizeInBytes(),
+            greaterThan(0L)
+        );
+    }
+
+    public void testInvalidationWithIndicesRequestCache() throws Exception {
+        internalCluster().startNode(
+            Settings.builder()
+                .put(defaultSettings(DEFAULT_CACHE_SIZE_IN_BYTES, null))
+                .put(INDICES_CACHE_CLEAN_INTERVAL_SETTING.getKey(), new TimeValue(1))
+                .build()
+        );
+        Client client = client();
+        assertAcked(
+            client.admin()
+                .indices()
+                .prepareCreate("index")
+                .setMapping("k", "type=keyword")
+                .setSettings(
+                    Settings.builder()
+                        .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
+                        .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                        .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                        .put("index.refresh_interval", -1)
+                )
+                .get()
+        );
+        int numberOfIndexedItems = randomIntBetween(5, 10);
+        for (int iterator = 0; iterator < numberOfIndexedItems; iterator++) {
+            indexRandom(true, client.prepareIndex("index").setSource("k" + iterator, "hello" + iterator));
+        }
+        ensureSearchable("index");
+        refreshAndWaitForReplication();
+        // Force merge the index to ensure there can be no background merges during the subsequent searches that would invalidate the cache
+        ForceMergeResponse forceMergeResponse = client.admin().indices().prepareForceMerge("index").setFlush(true).get();
+        OpenSearchAssertions.assertAllSuccessful(forceMergeResponse);
+        long perQuerySizeInCacheInBytes = -1;
+        for (int iterator = 0; iterator < numberOfIndexedItems; iterator++) {
+            SearchResponse resp = client.prepareSearch("index")
+                .setRequestCache(true)
+                .setQuery(QueryBuilders.termQuery("k" + iterator, "hello" + iterator))
+                .get();
+            if (perQuerySizeInCacheInBytes == -1) {
+                RequestCacheStats requestCacheStats = getRequestCacheStats(client, "index");
+                perQuerySizeInCacheInBytes = requestCacheStats.getMemorySizeInBytes();
+            }
+            assertSearchResponse(resp);
+        }
+        RequestCacheStats requestCacheStats = getRequestCacheStats(client, "index");
+        assertEquals(numberOfIndexedItems, requestCacheStats.getMissCount());
+        assertEquals(0, requestCacheStats.getHitCount());
+        assertEquals(0, requestCacheStats.getEvictions());
+        assertEquals(perQuerySizeInCacheInBytes * numberOfIndexedItems, requestCacheStats.getMemorySizeInBytes());
+        for (int iterator = 0; iterator < numberOfIndexedItems; iterator++) {
+            SearchResponse resp = client.prepareSearch("index")
+                .setRequestCache(true)
+                .setQuery(QueryBuilders.termQuery("k" + iterator, "hello" + iterator))
+                .get();
+            assertSearchResponse(resp);
+        }
+        requestCacheStats = getRequestCacheStats(client, "index");
+        assertEquals(numberOfIndexedItems, requestCacheStats.getHitCount());
+        assertEquals(numberOfIndexedItems, requestCacheStats.getMissCount());
+        assertEquals(perQuerySizeInCacheInBytes * numberOfIndexedItems, requestCacheStats.getMemorySizeInBytes());
+        assertEquals(0, requestCacheStats.getEvictions());
+        // Explicit refresh would invalidate cache entries.
+        refreshAndWaitForReplication();
+        assertBusy(() -> {
+            // Explicit refresh should clear up cache entries
+            assertTrue(getRequestCacheStats(client, "index").getMemorySizeInBytes() == 0);
+        }, 1, TimeUnit.SECONDS);
+        requestCacheStats = getRequestCacheStats(client, "index");
+        assertEquals(0, requestCacheStats.getMemorySizeInBytes());
+        // Hits and misses stats shouldn't get cleared up.
+        assertEquals(numberOfIndexedItems, requestCacheStats.getHitCount());
+        assertEquals(numberOfIndexedItems, requestCacheStats.getMissCount());
+    }
+
+    public void testExplicitCacheClearWithIndicesRequestCache() throws Exception {
+        internalCluster().startNode(
+            Settings.builder()
+                .put(defaultSettings(DEFAULT_CACHE_SIZE_IN_BYTES, null))
+                .put(INDICES_CACHE_CLEAN_INTERVAL_SETTING.getKey(), new TimeValue(1))
+                .build()
+        );
+        Client client = client();
+        assertAcked(
+            client.admin()
+                .indices()
+                .prepareCreate("index")
+                .setMapping("k", "type=keyword")
+                .setSettings(
+                    Settings.builder()
+                        .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
+                        .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                        .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                        .put("index.refresh_interval", -1)
+                )
+                .get()
+        );
+        int numberOfIndexedItems = randomIntBetween(5, 10);
+        for (int iterator = 0; iterator < numberOfIndexedItems; iterator++) {
+            indexRandom(true, client.prepareIndex("index").setSource("k" + iterator, "hello" + iterator));
+        }
+        ensureSearchable("index");
+        refreshAndWaitForReplication();
+        // Force merge the index to ensure there can be no background merges during the subsequent searches that would invalidate the cache
+        ForceMergeResponse forceMergeResponse = client.admin().indices().prepareForceMerge("index").setFlush(true).get();
+        OpenSearchAssertions.assertAllSuccessful(forceMergeResponse);
+
+        long perQuerySizeInCacheInBytes = -1;
+        for (int iterator = 0; iterator < numberOfIndexedItems; iterator++) {
+            SearchResponse resp = client.prepareSearch("index")
+                .setRequestCache(true)
+                .setQuery(QueryBuilders.termQuery("k" + iterator, "hello" + iterator))
+                .get();
+            if (perQuerySizeInCacheInBytes == -1) {
+                RequestCacheStats requestCacheStats = getRequestCacheStats(client, "index");
+                perQuerySizeInCacheInBytes = requestCacheStats.getMemorySizeInBytes();
+            }
+            assertSearchResponse(resp);
+        }
+        RequestCacheStats requestCacheStats = getRequestCacheStats(client, "index");
+        assertEquals(numberOfIndexedItems, requestCacheStats.getMissCount());
+        assertEquals(0, requestCacheStats.getHitCount());
+        assertEquals(0, requestCacheStats.getEvictions());
+        assertEquals(perQuerySizeInCacheInBytes * numberOfIndexedItems, requestCacheStats.getMemorySizeInBytes());
+
+        // Explicit clear the cache.
+        ClearIndicesCacheRequest request = new ClearIndicesCacheRequest("index");
+        ClearIndicesCacheResponse response = client.admin().indices().clearCache(request).get();
+        assertNoFailures(response);
+
+        assertBusy(() -> {
+            // All entries should get cleared up.
+            assertTrue(getRequestCacheStats(client, "index").getMemorySizeInBytes() == 0);
+        }, 1, TimeUnit.SECONDS);
+    }
+
+    public void testEvictionsFlowWithExpirationTime() throws Exception {
+        internalCluster().startNode(
+            Settings.builder()
+                .put(defaultSettings(DEFAULT_CACHE_SIZE_IN_BYTES, new TimeValue(0))) // Immediately evict items after
+                // access
+                .put(INDICES_CACHE_CLEAN_INTERVAL_SETTING.getKey(), new TimeValue(1))
+                .build()
+        );
+        Client client = client();
+        assertAcked(
+            client.admin()
+                .indices()
+                .prepareCreate("index")
+                .setMapping("k", "type=keyword")
+                .setSettings(
+                    Settings.builder()
+                        .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
+                        .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                        .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                        .put("index.refresh_interval", -1)
+                )
+                .get()
+        );
+        int numberOfIndexedItems = 2;// randomIntBetween(5, 10);
+        for (int iterator = 0; iterator < numberOfIndexedItems; iterator++) {
+            indexRandom(true, client.prepareIndex("index").setSource("k" + iterator, "hello" + iterator));
+        }
+        ensureSearchable("index");
+        refreshAndWaitForReplication();
+        // Force merge the index to ensure there can be no background merges during the subsequent searches that would invalidate the cache
+        ForceMergeResponse forceMergeResponse = client.admin().indices().prepareForceMerge("index").setFlush(true).get();
+        OpenSearchAssertions.assertAllSuccessful(forceMergeResponse);
+
+        long perQuerySizeInCacheInBytes = -1;
+        for (int iterator = 0; iterator < numberOfIndexedItems; iterator++) {
+            SearchResponse resp = client.prepareSearch("index")
+                .setRequestCache(true)
+                .setQuery(QueryBuilders.termQuery("k" + iterator, "hello" + iterator))
+                .get();
+            if (perQuerySizeInCacheInBytes == -1) {
+                RequestCacheStats requestCacheStats = getRequestCacheStats(client, "index");
+                perQuerySizeInCacheInBytes = requestCacheStats.getMemorySizeInBytes();
+            }
+            assertSearchResponse(resp);
+        }
+        RequestCacheStats requestCacheStats = getRequestCacheStats(client, "index");
+        assertEquals(0, requestCacheStats.getHitCount());
+        assertEquals(numberOfIndexedItems, requestCacheStats.getMissCount());
+        assertEquals(perQuerySizeInCacheInBytes * numberOfIndexedItems, requestCacheStats.getMemorySizeInBytes());
+        assertEquals(0, requestCacheStats.getEvictions());
+
+        for (int iterator = 0; iterator < numberOfIndexedItems; iterator++) {
+            SearchResponse resp = client.prepareSearch("index")
+                .setRequestCache(true)
+                .setQuery(QueryBuilders.termQuery("k" + iterator, "hello" + iterator))
+                .get();
+            assertSearchResponse(resp);
+        }
+        requestCacheStats = getRequestCacheStats(client, "index");
+        // Now that we have access the entries, they should expire after 1ms. So lets wait and verify that cache gets
+        // cleared up.
+        assertBusy(() -> {
+            // Explicit refresh should clear up cache entries
+            assertTrue(getRequestCacheStats(client, "index").getMemorySizeInBytes() == 0);
+        }, 10, TimeUnit.MILLISECONDS);
+        // Validate hit and miss count.
+        assertEquals(numberOfIndexedItems, requestCacheStats.getHitCount());
+        assertEquals(numberOfIndexedItems, requestCacheStats.getMissCount());
+    }
+
+    private RequestCacheStats getRequestCacheStats(Client client, String indexName) {
+        return client.admin().indices().prepareStats(indexName).setRequestCache(true).get().getTotal().getRequestCache();
+    }
 }
diff --git a/plugins/cache-ehcache/src/main/java/org/opensearch/cache/EhcacheDiskCacheSettings.java b/plugins/cache-ehcache/src/main/java/org/opensearch/cache/EhcacheDiskCacheSettings.java
index 837fd6b268ce6..d173155080f6a 100644
--- a/plugins/cache-ehcache/src/main/java/org/opensearch/cache/EhcacheDiskCacheSettings.java
+++ b/plugins/cache-ehcache/src/main/java/org/opensearch/cache/EhcacheDiskCacheSettings.java
@@ -23,6 +23,11 @@
  */
 public class EhcacheDiskCacheSettings {
 
+    /**
+     * Default cache size in bytes ie 1gb.
+     */
+    public static final long DEFAULT_CACHE_SIZE_IN_BYTES = 1073741824L;
+
     /**
      * Ehcache disk write minimum threads for its pool
      *
@@ -99,7 +104,7 @@ public class EhcacheDiskCacheSettings {
      */
     public static final Setting.AffixSetting<Long> DISK_CACHE_MAX_SIZE_IN_BYTES_SETTING = Setting.suffixKeySetting(
         EhcacheDiskCache.EhcacheDiskCacheFactory.EHCACHE_DISK_CACHE_NAME + ".max_size_in_bytes",
-        (key) -> Setting.longSetting(key, 1073741824L, NodeScope)
+        (key) -> Setting.longSetting(key, DEFAULT_CACHE_SIZE_IN_BYTES, NodeScope)
     );
 
     /**
diff --git a/plugins/cache-ehcache/src/main/java/org/opensearch/cache/store/disk/EhcacheDiskCache.java b/plugins/cache-ehcache/src/main/java/org/opensearch/cache/store/disk/EhcacheDiskCache.java
index edb2c900be46c..9a4dce1067b61 100644
--- a/plugins/cache-ehcache/src/main/java/org/opensearch/cache/store/disk/EhcacheDiskCache.java
+++ b/plugins/cache-ehcache/src/main/java/org/opensearch/cache/store/disk/EhcacheDiskCache.java
@@ -17,15 +17,20 @@
 import org.opensearch.common.annotation.ExperimentalApi;
 import org.opensearch.common.cache.CacheType;
 import org.opensearch.common.cache.ICache;
+import org.opensearch.common.cache.ICacheKey;
 import org.opensearch.common.cache.LoadAwareCacheLoader;
 import org.opensearch.common.cache.RemovalListener;
 import org.opensearch.common.cache.RemovalNotification;
 import org.opensearch.common.cache.RemovalReason;
+import org.opensearch.common.cache.serializer.ICacheKeySerializer;
 import org.opensearch.common.cache.serializer.Serializer;
+import org.opensearch.common.cache.stats.CacheStatsHolder;
+import org.opensearch.common.cache.stats.DefaultCacheStatsHolder;
+import org.opensearch.common.cache.stats.ImmutableCacheStatsHolder;
+import org.opensearch.common.cache.stats.NoopCacheStatsHolder;
 import org.opensearch.common.cache.store.builders.ICacheBuilder;
 import org.opensearch.common.cache.store.config.CacheConfig;
 import org.opensearch.common.collect.Tuple;
-import org.opensearch.common.metrics.CounterMetric;
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
@@ -40,6 +45,7 @@
 import java.time.Duration;
 import java.util.Arrays;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Map;
 import java.util.NoSuchElementException;
 import java.util.Objects;
@@ -49,6 +55,7 @@
 import java.util.concurrent.ExecutionException;
 import java.util.function.BiFunction;
 import java.util.function.Supplier;
+import java.util.function.ToLongBiFunction;
 
 import org.ehcache.Cache;
 import org.ehcache.CachePersistenceException;
@@ -101,21 +108,20 @@ public class EhcacheDiskCache<K, V> implements ICache<K, V> {
     private final PersistentCacheManager cacheManager;
 
     // Disk cache. Using ByteArrayWrapper to compare two byte[] by values rather than the default reference checks
-    private Cache<K, ByteArrayWrapper> cache;
+    @SuppressWarnings({ "rawtypes" }) // We have to use the raw type as there's no way to pass the "generic class" to ehcache
+    private Cache<ICacheKey, ByteArrayWrapper> cache;
     private final long maxWeightInBytes;
     private final String storagePath;
     private final Class<K> keyType;
     private final Class<V> valueType;
     private final TimeValue expireAfterAccess;
+    private final CacheStatsHolder cacheStatsHolder;
     private final EhCacheEventListener ehCacheEventListener;
     private final String threadPoolAlias;
     private final Settings settings;
-    private final RemovalListener<K, V> removalListener;
+    private final RemovalListener<ICacheKey<K>, V> removalListener;
     private final CacheType cacheType;
     private final String diskCacheAlias;
-    // TODO: Move count to stats once those changes are ready.
-    private final CounterMetric entries = new CounterMetric();
-
     private final Serializer<K, byte[]> keySerializer;
     private final Serializer<V, byte[]> valueSerializer;
 
@@ -123,7 +129,7 @@ public class EhcacheDiskCache<K, V> implements ICache<K, V> {
      * Used in computeIfAbsent to synchronize loading of a given key. This is needed as ehcache doesn't provide a
      * computeIfAbsent method.
      */
-    Map<K, CompletableFuture<Tuple<K, V>>> completableFutureMap = new ConcurrentHashMap<>();
+    Map<ICacheKey<K>, CompletableFuture<Tuple<ICacheKey<K>, V>>> completableFutureMap = new ConcurrentHashMap<>();
 
     private EhcacheDiskCache(Builder<K, V> builder) {
         this.keyType = Objects.requireNonNull(builder.keyType, "Key type shouldn't be null");
@@ -154,31 +160,45 @@ private EhcacheDiskCache(Builder<K, V> builder) {
         this.cacheManager = buildCacheManager();
         Objects.requireNonNull(builder.getRemovalListener(), "Removal listener can't be null");
         this.removalListener = builder.getRemovalListener();
-        this.ehCacheEventListener = new EhCacheEventListener(builder.getRemovalListener());
+        Objects.requireNonNull(builder.getWeigher(), "Weigher can't be null");
+        this.ehCacheEventListener = new EhCacheEventListener(builder.getRemovalListener(), builder.getWeigher());
         this.cache = buildCache(Duration.ofMillis(expireAfterAccess.getMillis()), builder);
+        List<String> dimensionNames = Objects.requireNonNull(builder.dimensionNames, "Dimension names can't be null");
+        if (builder.getStatsTrackingEnabled()) {
+            // If this cache is being used, FeatureFlags.PLUGGABLE_CACHE is already on, so we can always use the DefaultCacheStatsHolder
+            // unless statsTrackingEnabled is explicitly set to false in CacheConfig.
+            this.cacheStatsHolder = new DefaultCacheStatsHolder(dimensionNames, EhcacheDiskCacheFactory.EHCACHE_DISK_CACHE_NAME);
+        } else {
+            this.cacheStatsHolder = NoopCacheStatsHolder.getInstance();
+        }
     }
 
-    private Cache<K, ByteArrayWrapper> buildCache(Duration expireAfterAccess, Builder<K, V> builder) {
+    @SuppressWarnings({ "rawtypes" })
+    private Cache<ICacheKey, ByteArrayWrapper> buildCache(Duration expireAfterAccess, Builder<K, V> builder) {
         try {
             return this.cacheManager.createCache(
                 this.diskCacheAlias,
                 CacheConfigurationBuilder.newCacheConfigurationBuilder(
-                    this.keyType,
+                    ICacheKey.class,
                     ByteArrayWrapper.class,
                     ResourcePoolsBuilder.newResourcePoolsBuilder().disk(maxWeightInBytes, MemoryUnit.B)
                 ).withExpiry(new ExpiryPolicy<>() {
                     @Override
-                    public Duration getExpiryForCreation(K key, ByteArrayWrapper value) {
+                    public Duration getExpiryForCreation(ICacheKey key, ByteArrayWrapper value) {
                         return INFINITE;
                     }
 
                     @Override
-                    public Duration getExpiryForAccess(K key, Supplier<? extends ByteArrayWrapper> value) {
+                    public Duration getExpiryForAccess(ICacheKey key, Supplier<? extends ByteArrayWrapper> value) {
                         return expireAfterAccess;
                     }
 
                     @Override
-                    public Duration getExpiryForUpdate(K key, Supplier<? extends ByteArrayWrapper> oldValue, ByteArrayWrapper newValue) {
+                    public Duration getExpiryForUpdate(
+                        ICacheKey key,
+                        Supplier<? extends ByteArrayWrapper> oldValue,
+                        ByteArrayWrapper newValue
+                    ) {
                         return INFINITE;
                     }
                 })
@@ -192,7 +212,7 @@ public Duration getExpiryForUpdate(K key, Supplier<? extends ByteArrayWrapper> o
                             (Integer) EhcacheDiskCacheSettings.getSettingListForCacheType(cacheType).get(DISK_SEGMENT_KEY).get(settings)
                         )
                     )
-                    .withKeySerializer(new KeySerializerWrapper<K>(keySerializer))
+                    .withKeySerializer(new KeySerializerWrapper(keySerializer))
                     .withValueSerializer(new ByteArrayWrapperSerializer())
                 // We pass ByteArrayWrapperSerializer as ehcache's value serializer. If V is an interface, and we pass its
                 // serializer directly to ehcache, ehcache requires the classes match exactly before/after serialization.
@@ -225,7 +245,7 @@ private CacheEventListenerConfigurationBuilder getListenerConfiguration(Builder<
     }
 
     // Package private for testing
-    Map<K, CompletableFuture<Tuple<K, V>>> getCompletableFutureMap() {
+    Map<ICacheKey<K>, CompletableFuture<Tuple<ICacheKey<K>, V>>> getCompletableFutureMap() {
         return completableFutureMap;
     }
 
@@ -254,7 +274,7 @@ private PersistentCacheManager buildCacheManager() {
     }
 
     @Override
-    public V get(K key) {
+    public V get(ICacheKey<K> key) {
         if (key == null) {
             throw new IllegalArgumentException("Key passed to ehcache disk cache was null.");
         }
@@ -264,6 +284,11 @@ public V get(K key) {
         } catch (CacheLoadingException ex) {
             throw new OpenSearchException("Exception occurred while trying to fetch item from ehcache disk cache");
         }
+        if (value != null) {
+            cacheStatsHolder.incrementHits(key.dimensions);
+        } else {
+            cacheStatsHolder.incrementMisses(key.dimensions);
+        }
         return value;
     }
 
@@ -273,7 +298,7 @@ public V get(K key) {
      * @param value Type of value.
      */
     @Override
-    public void put(K key, V value) {
+    public void put(ICacheKey<K> key, V value) {
         try {
             cache.put(key, serializeValue(value));
         } catch (CacheWritingException ex) {
@@ -289,26 +314,31 @@ public void put(K key, V value) {
      * @throws Exception when either internal get or put calls fail.
      */
     @Override
-    public V computeIfAbsent(K key, LoadAwareCacheLoader<K, V> loader) throws Exception {
-        // Ehache doesn't provide any computeIfAbsent function. Exposes putIfAbsent but that works differently and is
+    public V computeIfAbsent(ICacheKey<K> key, LoadAwareCacheLoader<ICacheKey<K>, V> loader) throws Exception {
+        // Ehcache doesn't provide any computeIfAbsent function. Exposes putIfAbsent but that works differently and is
         // not performant in case there are multiple concurrent request for same key. Below is our own custom
         // implementation of computeIfAbsent on top of ehcache. Inspired by OpenSearch Cache implementation.
         V value = deserializeValue(cache.get(key));
         if (value == null) {
             value = compute(key, loader);
         }
+        if (!loader.isLoaded()) {
+            cacheStatsHolder.incrementHits(key.dimensions);
+        } else {
+            cacheStatsHolder.incrementMisses(key.dimensions);
+        }
         return value;
     }
 
-    private V compute(K key, LoadAwareCacheLoader<K, V> loader) throws Exception {
+    private V compute(ICacheKey<K> key, LoadAwareCacheLoader<ICacheKey<K>, V> loader) throws Exception {
         // A future that returns a pair of key/value.
-        CompletableFuture<Tuple<K, V>> completableFuture = new CompletableFuture<>();
+        CompletableFuture<Tuple<ICacheKey<K>, V>> completableFuture = new CompletableFuture<>();
         // Only one of the threads will succeed putting a future into map for the same key.
         // Rest will fetch existing future.
-        CompletableFuture<Tuple<K, V>> future = completableFutureMap.putIfAbsent(key, completableFuture);
+        CompletableFuture<Tuple<ICacheKey<K>, V>> future = completableFutureMap.putIfAbsent(key, completableFuture);
         // Handler to handle results post processing. Takes a tuple<key, value> or exception as an input and returns
         // the value. Also before returning value, puts the value in cache.
-        BiFunction<Tuple<K, V>, Throwable, V> handler = (pair, ex) -> {
+        BiFunction<Tuple<ICacheKey<K>, V>, Throwable, V> handler = (pair, ex) -> {
             V value = null;
             if (pair != null) {
                 cache.put(pair.v1(), serializeValue(pair.v2()));
@@ -358,9 +388,14 @@ private V compute(K key, LoadAwareCacheLoader<K, V> loader) throws Exception {
      * @param key key to be invalidated.
      */
     @Override
-    public void invalidate(K key) {
+    public void invalidate(ICacheKey<K> key) {
         try {
-            cache.remove(key);
+            if (key.getDropStatsForDimensions()) {
+                cacheStatsHolder.removeDimensions(key.dimensions);
+            }
+            if (key.key != null) {
+                cache.remove(key);
+            }
         } catch (CacheWritingException ex) {
             // Handle
             throw new RuntimeException(ex);
@@ -371,7 +406,7 @@ public void invalidate(K key) {
     @Override
     public void invalidateAll() {
         cache.clear();
-        this.entries.dec(this.entries.count()); // reset to zero.
+        cacheStatsHolder.reset();
     }
 
     /**
@@ -379,17 +414,18 @@ public void invalidateAll() {
      * @return Iterable
      */
     @Override
-    public Iterable<K> keys() {
+    public Iterable<ICacheKey<K>> keys() {
         return () -> new EhCacheKeyIterator<>(cache.iterator());
     }
 
     /**
      * Gives the current count of keys in disk cache.
+     * If enableStatsTracking is set to false in the builder, always returns 0.
      * @return current count of keys
      */
     @Override
     public long count() {
-        return entries.count();
+        return cacheStatsHolder.count();
     }
 
     @Override
@@ -416,15 +452,26 @@ public void close() {
         }
     }
 
+    /**
+     * Relevant stats for this cache, aggregated by levels.
+     * @param levels The levels to aggregate by.
+     * @return ImmutableCacheStatsHolder
+     */
+    @Override
+    public ImmutableCacheStatsHolder stats(String[] levels) {
+        return cacheStatsHolder.getImmutableCacheStatsHolder(levels);
+    }
+
     /**
      * This iterator wraps ehCache iterator and only iterates over its keys.
      * @param <K> Type of key
      */
-    class EhCacheKeyIterator<K> implements Iterator<K> {
+    @SuppressWarnings({ "rawtypes", "unchecked" })
+    class EhCacheKeyIterator<K> implements Iterator<ICacheKey<K>> {
 
-        Iterator<Cache.Entry<K, ByteArrayWrapper>> iterator;
+        Iterator<Cache.Entry<ICacheKey, ByteArrayWrapper>> iterator;
 
-        EhCacheKeyIterator(Iterator<Cache.Entry<K, ByteArrayWrapper>> iterator) {
+        EhCacheKeyIterator(Iterator<Cache.Entry<ICacheKey, ByteArrayWrapper>> iterator) {
             this.iterator = iterator;
         }
 
@@ -434,7 +481,7 @@ public boolean hasNext() {
         }
 
         @Override
-        public K next() {
+        public ICacheKey<K> next() {
             if (!hasNext()) {
                 throw new NoSuchElementException();
             }
@@ -450,43 +497,60 @@ public void remove() {
     /**
      * Wrapper over Ehcache original listener to listen to desired events and notify desired subscribers.
      */
-    class EhCacheEventListener implements CacheEventListener<K, ByteArrayWrapper> {
-
-        private final RemovalListener<K, V> removalListener;
+    class EhCacheEventListener implements CacheEventListener<ICacheKey<K>, ByteArrayWrapper> {
+        private final RemovalListener<ICacheKey<K>, V> removalListener;
+        private ToLongBiFunction<ICacheKey<K>, V> weigher;
 
-        EhCacheEventListener(RemovalListener<K, V> removalListener) {
+        EhCacheEventListener(RemovalListener<ICacheKey<K>, V> removalListener, ToLongBiFunction<ICacheKey<K>, V> weigher) {
             this.removalListener = removalListener;
+            this.weigher = weigher;
+        }
+
+        private long getOldValuePairSize(CacheEvent<? extends ICacheKey<K>, ? extends ByteArrayWrapper> event) {
+            return weigher.applyAsLong(event.getKey(), deserializeValue(event.getOldValue()));
+        }
+
+        private long getNewValuePairSize(CacheEvent<? extends ICacheKey<K>, ? extends ByteArrayWrapper> event) {
+            return weigher.applyAsLong(event.getKey(), deserializeValue(event.getNewValue()));
         }
 
         @Override
-        public void onEvent(CacheEvent<? extends K, ? extends ByteArrayWrapper> event) {
+        public void onEvent(CacheEvent<? extends ICacheKey<K>, ? extends ByteArrayWrapper> event) {
             switch (event.getType()) {
                 case CREATED:
-                    entries.inc();
+                    cacheStatsHolder.incrementItems(event.getKey().dimensions);
+                    cacheStatsHolder.incrementSizeInBytes(event.getKey().dimensions, getNewValuePairSize(event));
                     assert event.getOldValue() == null;
                     break;
                 case EVICTED:
                     this.removalListener.onRemoval(
                         new RemovalNotification<>(event.getKey(), deserializeValue(event.getOldValue()), RemovalReason.EVICTED)
                     );
-                    entries.dec();
+                    cacheStatsHolder.decrementItems(event.getKey().dimensions);
+                    cacheStatsHolder.decrementSizeInBytes(event.getKey().dimensions, getOldValuePairSize(event));
+                    cacheStatsHolder.incrementEvictions(event.getKey().dimensions);
                     assert event.getNewValue() == null;
                     break;
                 case REMOVED:
-                    entries.dec();
                     this.removalListener.onRemoval(
                         new RemovalNotification<>(event.getKey(), deserializeValue(event.getOldValue()), RemovalReason.EXPLICIT)
                     );
+                    cacheStatsHolder.decrementItems(event.getKey().dimensions);
+                    cacheStatsHolder.decrementSizeInBytes(event.getKey().dimensions, getOldValuePairSize(event));
                     assert event.getNewValue() == null;
                     break;
                 case EXPIRED:
                     this.removalListener.onRemoval(
                         new RemovalNotification<>(event.getKey(), deserializeValue(event.getOldValue()), RemovalReason.INVALIDATED)
                     );
-                    entries.dec();
+                    cacheStatsHolder.decrementItems(event.getKey().dimensions);
+                    cacheStatsHolder.decrementSizeInBytes(event.getKey().dimensions, getOldValuePairSize(event));
                     assert event.getNewValue() == null;
                     break;
                 case UPDATED:
+                    long newSize = getNewValuePairSize(event);
+                    long oldSize = getOldValuePairSize(event);
+                    cacheStatsHolder.incrementSizeInBytes(event.getKey().dimensions, newSize - oldSize);
                     break;
                 default:
                     break;
@@ -495,13 +559,14 @@ public void onEvent(CacheEvent<? extends K, ? extends ByteArrayWrapper> event) {
     }
 
     /**
-     * Wrapper over Serializer which is compatible with ehcache's serializer requirements.
+     * Wrapper over ICacheKeySerializer which is compatible with ehcache's serializer requirements.
      */
-    private class KeySerializerWrapper<T> implements org.ehcache.spi.serialization.Serializer<T> {
-        private Serializer<T, byte[]> serializer;
+    @SuppressWarnings({ "rawtypes", "unchecked" })
+    private class KeySerializerWrapper implements org.ehcache.spi.serialization.Serializer<ICacheKey> {
+        private ICacheKeySerializer<K> serializer;
 
-        public KeySerializerWrapper(Serializer<T, byte[]> keySerializer) {
-            this.serializer = keySerializer;
+        public KeySerializerWrapper(Serializer<K, byte[]> internalKeySerializer) {
+            this.serializer = new ICacheKeySerializer<>(internalKeySerializer);
         }
 
         // This constructor must be present, but does not have to work as we are not actually persisting the disk
@@ -510,19 +575,19 @@ public KeySerializerWrapper(Serializer<T, byte[]> keySerializer) {
         public KeySerializerWrapper(ClassLoader classLoader, FileBasedPersistenceContext persistenceContext) {}
 
         @Override
-        public ByteBuffer serialize(T object) throws SerializerException {
+        public ByteBuffer serialize(ICacheKey object) throws SerializerException {
             return ByteBuffer.wrap(serializer.serialize(object));
         }
 
         @Override
-        public T read(ByteBuffer binary) throws ClassNotFoundException, SerializerException {
+        public ICacheKey<K> read(ByteBuffer binary) throws ClassNotFoundException, SerializerException {
             byte[] arr = new byte[binary.remaining()];
             binary.get(arr);
             return serializer.deserialize(arr);
         }
 
         @Override
-        public boolean equals(T object, ByteBuffer binary) throws ClassNotFoundException, SerializerException {
+        public boolean equals(ICacheKey object, ByteBuffer binary) throws ClassNotFoundException, SerializerException {
             byte[] arr = new byte[binary.remaining()];
             binary.get(arr);
             return serializer.equals(object, arr);
@@ -566,8 +631,7 @@ public boolean equals(ByteArrayWrapper object, ByteBuffer binary) throws ClassNo
      * @return the serialized value
      */
     private ByteArrayWrapper serializeValue(V value) {
-        ByteArrayWrapper result = new ByteArrayWrapper(valueSerializer.serialize(value));
-        return result;
+        return new ByteArrayWrapper(valueSerializer.serialize(value));
     }
 
     /**
@@ -625,6 +689,8 @@ public <K, V> ICache<K, V> create(CacheConfig<K, V> config, CacheType cacheType,
                 .setValueType(config.getValueType())
                 .setKeySerializer(keySerializer)
                 .setValueSerializer(valueSerializer)
+                .setDimensionNames(config.getDimensionNames())
+                .setWeigher(config.getWeigher())
                 .setRemovalListener(config.getRemovalListener())
                 .setExpireAfterAccess((TimeValue) settingList.get(DISK_CACHE_EXPIRE_AFTER_ACCESS_KEY).get(settings))
                 .setMaximumWeightInBytes((Long) settingList.get(DISK_MAX_SIZE_IN_BYTES_KEY).get(settings))
@@ -658,6 +724,7 @@ public static class Builder<K, V> extends ICacheBuilder<K, V> {
         private Class<K> keyType;
 
         private Class<V> valueType;
+        private List<String> dimensionNames;
         private Serializer<K, byte[]> keySerializer;
         private Serializer<V, byte[]> valueSerializer;
 
@@ -736,6 +803,16 @@ public Builder<K, V> setIsEventListenerModeSync(boolean isEventListenerModeSync)
             return this;
         }
 
+        /**
+         * Sets the allowed dimension names for keys that will enter this cache.
+         * @param dimensionNames A list of dimension names this cache will accept
+         * @return builder
+         */
+        public Builder<K, V> setDimensionNames(List<String> dimensionNames) {
+            this.dimensionNames = dimensionNames;
+            return this;
+        }
+
         /**
          * Sets the key serializer for this cache.
          * @param keySerializer the key serializer
@@ -764,7 +841,7 @@ public EhcacheDiskCache<K, V> build() {
 
     /**
      * A wrapper over byte[], with equals() that works using Arrays.equals().
-     * Necessary due to a bug in Ehcache.
+     * Necessary due to a limitation in how Ehcache compares byte[].
      */
     static class ByteArrayWrapper {
         private final byte[] value;
diff --git a/plugins/cache-ehcache/src/test/java/org/opensearch/cache/store/disk/EhCacheDiskCacheTests.java b/plugins/cache-ehcache/src/test/java/org/opensearch/cache/store/disk/EhCacheDiskCacheTests.java
index 3a98ad2fef6b1..29551befd3e9f 100644
--- a/plugins/cache-ehcache/src/test/java/org/opensearch/cache/store/disk/EhCacheDiskCacheTests.java
+++ b/plugins/cache-ehcache/src/test/java/org/opensearch/cache/store/disk/EhCacheDiskCacheTests.java
@@ -14,11 +14,13 @@
 import org.opensearch.common.Randomness;
 import org.opensearch.common.cache.CacheType;
 import org.opensearch.common.cache.ICache;
+import org.opensearch.common.cache.ICacheKey;
 import org.opensearch.common.cache.LoadAwareCacheLoader;
 import org.opensearch.common.cache.RemovalListener;
 import org.opensearch.common.cache.RemovalNotification;
 import org.opensearch.common.cache.serializer.BytesReferenceSerializer;
 import org.opensearch.common.cache.serializer.Serializer;
+import org.opensearch.common.cache.stats.ImmutableCacheStats;
 import org.opensearch.common.cache.store.config.CacheConfig;
 import org.opensearch.common.metrics.CounterMetric;
 import org.opensearch.common.settings.Settings;
@@ -43,6 +45,7 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Phaser;
+import java.util.function.ToLongBiFunction;
 
 import static org.opensearch.cache.EhcacheDiskCacheSettings.DISK_LISTENER_MODE_SYNC_KEY;
 import static org.opensearch.cache.EhcacheDiskCacheSettings.DISK_MAX_SIZE_IN_BYTES_KEY;
@@ -53,10 +56,12 @@
 public class EhCacheDiskCacheTests extends OpenSearchSingleNodeTestCase {
 
     private static final int CACHE_SIZE_IN_BYTES = 1024 * 101;
+    private final String dimensionName = "shardId";
 
     public void testBasicGetAndPut() throws IOException {
         Settings settings = Settings.builder().build();
         MockRemovalListener<String, String> removalListener = new MockRemovalListener<>();
+        ToLongBiFunction<ICacheKey<String>, String> weigher = getWeigher();
         try (NodeEnvironment env = newNodeEnvironment(settings)) {
             ICache<String, String> ehcacheTest = new EhcacheDiskCache.Builder<String, String>().setThreadPoolAlias("ehcacheTest")
                 .setStoragePath(env.nodePaths()[0].indicesPath.toString() + "/request_cache")
@@ -65,32 +70,42 @@ public void testBasicGetAndPut() throws IOException {
                 .setValueType(String.class)
                 .setKeySerializer(new StringSerializer())
                 .setValueSerializer(new StringSerializer())
+                .setDimensionNames(List.of(dimensionName))
                 .setCacheType(CacheType.INDICES_REQUEST_CACHE)
                 .setSettings(settings)
                 .setExpireAfterAccess(TimeValue.MAX_VALUE)
                 .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES)
                 .setRemovalListener(removalListener)
+                .setWeigher(weigher)
                 .build();
             int randomKeys = randomIntBetween(10, 100);
+            long expectedSize = 0;
             Map<String, String> keyValueMap = new HashMap<>();
             for (int i = 0; i < randomKeys; i++) {
                 keyValueMap.put(UUID.randomUUID().toString(), UUID.randomUUID().toString());
             }
             for (Map.Entry<String, String> entry : keyValueMap.entrySet()) {
-                ehcacheTest.put(entry.getKey(), entry.getValue());
+                ICacheKey<String> iCacheKey = getICacheKey(entry.getKey());
+                ehcacheTest.put(iCacheKey, entry.getValue());
+                expectedSize += weigher.applyAsLong(iCacheKey, entry.getValue());
             }
             for (Map.Entry<String, String> entry : keyValueMap.entrySet()) {
-                String value = ehcacheTest.get(entry.getKey());
+                String value = ehcacheTest.get(getICacheKey(entry.getKey()));
                 assertEquals(entry.getValue(), value);
             }
+            assertEquals(randomKeys, ehcacheTest.stats().getTotalItems());
+            assertEquals(randomKeys, ehcacheTest.stats().getTotalHits());
+            assertEquals(expectedSize, ehcacheTest.stats().getTotalSizeInBytes());
             assertEquals(randomKeys, ehcacheTest.count());
 
             // Validate misses
             int expectedNumberOfMisses = randomIntBetween(10, 200);
             for (int i = 0; i < expectedNumberOfMisses; i++) {
-                ehcacheTest.get(UUID.randomUUID().toString());
+                ehcacheTest.get(getICacheKey(UUID.randomUUID().toString()));
             }
 
+            assertEquals(expectedNumberOfMisses, ehcacheTest.stats().getTotalMisses());
+
             ehcacheTest.close();
         }
     }
@@ -105,6 +120,8 @@ public void testBasicGetAndPutUsingFactory() throws IOException {
                     .setRemovalListener(removalListener)
                     .setKeySerializer(new StringSerializer())
                     .setValueSerializer(new StringSerializer())
+                    .setDimensionNames(List.of(dimensionName))
+                    .setWeigher(getWeigher())
                     .setSettings(
                         Settings.builder()
                             .put(
@@ -132,14 +149,14 @@ public void testBasicGetAndPutUsingFactory() throws IOException {
                 Map.of()
             );
             int randomKeys = randomIntBetween(10, 100);
-            Map<String, String> keyValueMap = new HashMap<>();
+            Map<ICacheKey<String>, String> keyValueMap = new HashMap<>();
             for (int i = 0; i < randomKeys; i++) {
-                keyValueMap.put(UUID.randomUUID().toString(), UUID.randomUUID().toString());
+                keyValueMap.put(getICacheKey(UUID.randomUUID().toString()), UUID.randomUUID().toString());
             }
-            for (Map.Entry<String, String> entry : keyValueMap.entrySet()) {
+            for (Map.Entry<ICacheKey<String>, String> entry : keyValueMap.entrySet()) {
                 ehcacheTest.put(entry.getKey(), entry.getValue());
             }
-            for (Map.Entry<String, String> entry : keyValueMap.entrySet()) {
+            for (Map.Entry<ICacheKey<String>, String> entry : keyValueMap.entrySet()) {
                 String value = ehcacheTest.get(entry.getKey());
                 assertEquals(entry.getValue(), value);
             }
@@ -148,7 +165,7 @@ public void testBasicGetAndPutUsingFactory() throws IOException {
             // Validate misses
             int expectedNumberOfMisses = randomIntBetween(10, 200);
             for (int i = 0; i < expectedNumberOfMisses; i++) {
-                ehcacheTest.get(UUID.randomUUID().toString());
+                ehcacheTest.get(getICacheKey(UUID.randomUUID().toString()));
             }
 
             ehcacheTest.close();
@@ -167,22 +184,24 @@ public void testConcurrentPut() throws Exception {
                 .setValueType(String.class)
                 .setKeySerializer(new StringSerializer())
                 .setValueSerializer(new StringSerializer())
+                .setDimensionNames(List.of(dimensionName))
                 .setCacheType(CacheType.INDICES_REQUEST_CACHE)
                 .setSettings(settings)
                 .setExpireAfterAccess(TimeValue.MAX_VALUE)
                 .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES)
                 .setRemovalListener(removalListener)
+                .setWeigher(getWeigher())
                 .build();
             int randomKeys = randomIntBetween(20, 100);
             Thread[] threads = new Thread[randomKeys];
             Phaser phaser = new Phaser(randomKeys + 1);
             CountDownLatch countDownLatch = new CountDownLatch(randomKeys);
-            Map<String, String> keyValueMap = new HashMap<>();
+            Map<ICacheKey<String>, String> keyValueMap = new HashMap<>();
             int j = 0;
             for (int i = 0; i < randomKeys; i++) {
-                keyValueMap.put(UUID.randomUUID().toString(), UUID.randomUUID().toString());
+                keyValueMap.put(getICacheKey(UUID.randomUUID().toString()), UUID.randomUUID().toString());
             }
-            for (Map.Entry<String, String> entry : keyValueMap.entrySet()) {
+            for (Map.Entry<ICacheKey<String>, String> entry : keyValueMap.entrySet()) {
                 threads[j] = new Thread(() -> {
                     phaser.arriveAndAwaitAdvance();
                     ehcacheTest.put(entry.getKey(), entry.getValue());
@@ -193,11 +212,12 @@ public void testConcurrentPut() throws Exception {
             }
             phaser.arriveAndAwaitAdvance(); // Will trigger parallel puts above.
             countDownLatch.await(); // Wait for all threads to finish
-            for (Map.Entry<String, String> entry : keyValueMap.entrySet()) {
+            for (Map.Entry<ICacheKey<String>, String> entry : keyValueMap.entrySet()) {
                 String value = ehcacheTest.get(entry.getKey());
                 assertEquals(entry.getValue(), value);
             }
             assertEquals(randomKeys, ehcacheTest.count());
+            assertEquals(randomKeys, ehcacheTest.stats().getTotalItems());
             ehcacheTest.close();
         }
     }
@@ -214,11 +234,13 @@ public void testEhcacheParallelGets() throws Exception {
                 .setValueType(String.class)
                 .setKeySerializer(new StringSerializer())
                 .setValueSerializer(new StringSerializer())
+                .setDimensionNames(List.of(dimensionName))
                 .setCacheType(CacheType.INDICES_REQUEST_CACHE)
                 .setSettings(settings)
                 .setExpireAfterAccess(TimeValue.MAX_VALUE)
                 .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES)
                 .setRemovalListener(removalListener)
+                .setWeigher(getWeigher())
                 .build();
             int randomKeys = randomIntBetween(20, 100);
             Thread[] threads = new Thread[randomKeys];
@@ -230,13 +252,13 @@ public void testEhcacheParallelGets() throws Exception {
                 keyValueMap.put(UUID.randomUUID().toString(), UUID.randomUUID().toString());
             }
             for (Map.Entry<String, String> entry : keyValueMap.entrySet()) {
-                ehcacheTest.put(entry.getKey(), entry.getValue());
+                ehcacheTest.put(getICacheKey(entry.getKey()), entry.getValue());
             }
             assertEquals(keyValueMap.size(), ehcacheTest.count());
             for (Map.Entry<String, String> entry : keyValueMap.entrySet()) {
                 threads[j] = new Thread(() -> {
                     phaser.arriveAndAwaitAdvance();
-                    assertEquals(entry.getValue(), ehcacheTest.get(entry.getKey()));
+                    assertEquals(entry.getValue(), ehcacheTest.get(getICacheKey(entry.getKey())));
                     countDownLatch.countDown();
                 });
                 threads[j].start();
@@ -244,6 +266,7 @@ public void testEhcacheParallelGets() throws Exception {
             }
             phaser.arriveAndAwaitAdvance(); // Will trigger parallel puts above.
             countDownLatch.await(); // Wait for all threads to finish
+            assertEquals(randomKeys, ehcacheTest.stats().getTotalHits());
             ehcacheTest.close();
         }
     }
@@ -259,11 +282,13 @@ public void testEhcacheKeyIterator() throws Exception {
                 .setValueType(String.class)
                 .setKeySerializer(new StringSerializer())
                 .setValueSerializer(new StringSerializer())
+                .setDimensionNames(List.of(dimensionName))
                 .setCacheType(CacheType.INDICES_REQUEST_CACHE)
                 .setSettings(settings)
                 .setExpireAfterAccess(TimeValue.MAX_VALUE)
                 .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES)
                 .setRemovalListener(new MockRemovalListener<>())
+                .setWeigher(getWeigher())
                 .build();
 
             int randomKeys = randomIntBetween(2, 100);
@@ -272,12 +297,12 @@ public void testEhcacheKeyIterator() throws Exception {
                 keyValueMap.put(UUID.randomUUID().toString(), UUID.randomUUID().toString());
             }
             for (Map.Entry<String, String> entry : keyValueMap.entrySet()) {
-                ehcacheTest.put(entry.getKey(), entry.getValue());
+                ehcacheTest.put(getICacheKey(entry.getKey()), entry.getValue());
             }
-            Iterator<String> keys = ehcacheTest.keys().iterator();
+            Iterator<ICacheKey<String>> keys = ehcacheTest.keys().iterator();
             int keysCount = 0;
             while (keys.hasNext()) {
-                String key = keys.next();
+                ICacheKey<String> key = keys.next();
                 keysCount++;
                 assertNotNull(ehcacheTest.get(key));
             }
@@ -289,6 +314,7 @@ public void testEhcacheKeyIterator() throws Exception {
     public void testEvictions() throws Exception {
         Settings settings = Settings.builder().build();
         MockRemovalListener<String, String> removalListener = new MockRemovalListener<>();
+        ToLongBiFunction<ICacheKey<String>, String> weigher = getWeigher();
         try (NodeEnvironment env = newNodeEnvironment(settings)) {
             ICache<String, String> ehcacheTest = new EhcacheDiskCache.Builder<String, String>().setDiskCacheAlias("test1")
                 .setStoragePath(env.nodePaths()[0].indicesPath.toString() + "/request_cache")
@@ -298,11 +324,13 @@ public void testEvictions() throws Exception {
                 .setValueType(String.class)
                 .setKeySerializer(new StringSerializer())
                 .setValueSerializer(new StringSerializer())
+                .setDimensionNames(List.of(dimensionName))
                 .setCacheType(CacheType.INDICES_REQUEST_CACHE)
                 .setSettings(settings)
                 .setExpireAfterAccess(TimeValue.MAX_VALUE)
                 .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES)
                 .setRemovalListener(removalListener)
+                .setWeigher(weigher)
                 .build();
 
             // Generate a string with 100 characters
@@ -311,9 +339,10 @@ public void testEvictions() throws Exception {
             // Trying to generate more than 100kb to cause evictions.
             for (int i = 0; i < 1000; i++) {
                 String key = "Key" + i;
-                ehcacheTest.put(key, value);
+                ehcacheTest.put(getICacheKey(key), value);
             }
             assertEquals(660, removalListener.evictionMetric.count());
+            assertEquals(660, ehcacheTest.stats().getTotalEvictions());
             ehcacheTest.close();
         }
     }
@@ -330,11 +359,13 @@ public void testComputeIfAbsentConcurrently() throws Exception {
                 .setValueType(String.class)
                 .setKeySerializer(new StringSerializer())
                 .setValueSerializer(new StringSerializer())
+                .setDimensionNames(List.of(dimensionName))
                 .setCacheType(CacheType.INDICES_REQUEST_CACHE)
                 .setSettings(settings)
                 .setExpireAfterAccess(TimeValue.MAX_VALUE)
                 .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES)
                 .setRemovalListener(removalListener)
+                .setWeigher(getWeigher())
                 .build();
 
             int numberOfRequest = 2;// randomIntBetween(200, 400);
@@ -344,12 +375,12 @@ public void testComputeIfAbsentConcurrently() throws Exception {
             Phaser phaser = new Phaser(numberOfRequest + 1);
             CountDownLatch countDownLatch = new CountDownLatch(numberOfRequest);
 
-            List<LoadAwareCacheLoader<String, String>> loadAwareCacheLoaderList = new CopyOnWriteArrayList<>();
+            List<LoadAwareCacheLoader<ICacheKey<String>, String>> loadAwareCacheLoaderList = new CopyOnWriteArrayList<>();
 
             // Try to hit different request with the same key concurrently. Verify value is only loaded once.
             for (int i = 0; i < numberOfRequest; i++) {
                 threads[i] = new Thread(() -> {
-                    LoadAwareCacheLoader<String, String> loadAwareCacheLoader = new LoadAwareCacheLoader<>() {
+                    LoadAwareCacheLoader<ICacheKey<String>, String> loadAwareCacheLoader = new LoadAwareCacheLoader<>() {
                         boolean isLoaded;
 
                         @Override
@@ -358,7 +389,7 @@ public boolean isLoaded() {
                         }
 
                         @Override
-                        public String load(String key) {
+                        public String load(ICacheKey<String> key) {
                             isLoaded = true;
                             return value;
                         }
@@ -366,7 +397,7 @@ public String load(String key) {
                     loadAwareCacheLoaderList.add(loadAwareCacheLoader);
                     phaser.arriveAndAwaitAdvance();
                     try {
-                        assertEquals(value, ehcacheTest.computeIfAbsent(key, loadAwareCacheLoader));
+                        assertEquals(value, ehcacheTest.computeIfAbsent(getICacheKey(key), loadAwareCacheLoader));
                     } catch (Exception e) {
                         throw new RuntimeException(e);
                     }
@@ -384,6 +415,9 @@ public String load(String key) {
             }
             assertEquals(1, numberOfTimesValueLoaded);
             assertEquals(0, ((EhcacheDiskCache) ehcacheTest).getCompletableFutureMap().size());
+            assertEquals(1, ehcacheTest.stats().getTotalMisses());
+            assertEquals(1, ehcacheTest.stats().getTotalItems());
+            assertEquals(numberOfRequest - 1, ehcacheTest.stats().getTotalHits());
             assertEquals(1, ehcacheTest.count());
             ehcacheTest.close();
         }
@@ -401,11 +435,13 @@ public void testComputeIfAbsentConcurrentlyAndThrowsException() throws Exception
                 .setValueType(String.class)
                 .setKeySerializer(new StringSerializer())
                 .setValueSerializer(new StringSerializer())
+                .setDimensionNames(List.of(dimensionName))
                 .setCacheType(CacheType.INDICES_REQUEST_CACHE)
                 .setSettings(settings)
                 .setExpireAfterAccess(TimeValue.MAX_VALUE)
                 .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES)
                 .setRemovalListener(removalListener)
+                .setWeigher(getWeigher())
                 .build();
 
             int numberOfRequest = randomIntBetween(200, 400);
@@ -414,12 +450,12 @@ public void testComputeIfAbsentConcurrentlyAndThrowsException() throws Exception
             Phaser phaser = new Phaser(numberOfRequest + 1);
             CountDownLatch countDownLatch = new CountDownLatch(numberOfRequest);
 
-            List<LoadAwareCacheLoader<String, String>> loadAwareCacheLoaderList = new CopyOnWriteArrayList<>();
+            List<LoadAwareCacheLoader<ICacheKey<String>, String>> loadAwareCacheLoaderList = new CopyOnWriteArrayList<>();
 
             // Try to hit different request with the same key concurrently. Loader throws exception.
             for (int i = 0; i < numberOfRequest; i++) {
                 threads[i] = new Thread(() -> {
-                    LoadAwareCacheLoader<String, String> loadAwareCacheLoader = new LoadAwareCacheLoader<>() {
+                    LoadAwareCacheLoader<ICacheKey<String>, String> loadAwareCacheLoader = new LoadAwareCacheLoader<>() {
                         boolean isLoaded;
 
                         @Override
@@ -428,14 +464,14 @@ public boolean isLoaded() {
                         }
 
                         @Override
-                        public String load(String key) throws Exception {
+                        public String load(ICacheKey<String> key) throws Exception {
                             isLoaded = true;
                             throw new RuntimeException("Exception");
                         }
                     };
                     loadAwareCacheLoaderList.add(loadAwareCacheLoader);
                     phaser.arriveAndAwaitAdvance();
-                    assertThrows(ExecutionException.class, () -> ehcacheTest.computeIfAbsent(key, loadAwareCacheLoader));
+                    assertThrows(ExecutionException.class, () -> ehcacheTest.computeIfAbsent(getICacheKey(key), loadAwareCacheLoader));
                     countDownLatch.countDown();
                 });
                 threads[i].start();
@@ -460,11 +496,13 @@ public void testComputeIfAbsentWithNullValueLoading() throws Exception {
                 .setValueType(String.class)
                 .setKeySerializer(new StringSerializer())
                 .setValueSerializer(new StringSerializer())
+                .setDimensionNames(List.of(dimensionName))
                 .setCacheType(CacheType.INDICES_REQUEST_CACHE)
                 .setSettings(settings)
                 .setExpireAfterAccess(TimeValue.MAX_VALUE)
                 .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES)
                 .setRemovalListener(removalListener)
+                .setWeigher(getWeigher())
                 .build();
 
             int numberOfRequest = randomIntBetween(200, 400);
@@ -473,12 +511,12 @@ public void testComputeIfAbsentWithNullValueLoading() throws Exception {
             Phaser phaser = new Phaser(numberOfRequest + 1);
             CountDownLatch countDownLatch = new CountDownLatch(numberOfRequest);
 
-            List<LoadAwareCacheLoader<String, String>> loadAwareCacheLoaderList = new CopyOnWriteArrayList<>();
+            List<LoadAwareCacheLoader<ICacheKey<String>, String>> loadAwareCacheLoaderList = new CopyOnWriteArrayList<>();
 
             // Try to hit different request with the same key concurrently. Loader throws exception.
             for (int i = 0; i < numberOfRequest; i++) {
                 threads[i] = new Thread(() -> {
-                    LoadAwareCacheLoader<String, String> loadAwareCacheLoader = new LoadAwareCacheLoader<>() {
+                    LoadAwareCacheLoader<ICacheKey<String>, String> loadAwareCacheLoader = new LoadAwareCacheLoader<>() {
                         boolean isLoaded;
 
                         @Override
@@ -487,7 +525,7 @@ public boolean isLoaded() {
                         }
 
                         @Override
-                        public String load(String key) throws Exception {
+                        public String load(ICacheKey<String> key) throws Exception {
                             isLoaded = true;
                             return null;
                         }
@@ -495,11 +533,11 @@ public String load(String key) throws Exception {
                     loadAwareCacheLoaderList.add(loadAwareCacheLoader);
                     phaser.arriveAndAwaitAdvance();
                     try {
-                        ehcacheTest.computeIfAbsent(key, loadAwareCacheLoader);
+                        ehcacheTest.computeIfAbsent(getICacheKey(key), loadAwareCacheLoader);
                     } catch (Exception ex) {
                         assertThat(ex.getCause(), instanceOf(NullPointerException.class));
                     }
-                    assertThrows(ExecutionException.class, () -> ehcacheTest.computeIfAbsent(key, loadAwareCacheLoader));
+                    assertThrows(ExecutionException.class, () -> ehcacheTest.computeIfAbsent(getICacheKey(key), loadAwareCacheLoader));
                     countDownLatch.countDown();
                 });
                 threads[i].start();
@@ -512,42 +550,119 @@ public String load(String key) throws Exception {
         }
     }
 
-    public void testEhcacheKeyIteratorWithRemove() throws IOException {
+    public void testMemoryTracking() throws Exception {
+        // Test all cases for EhCacheEventListener.onEvent and check stats memory usage is updated correctly
         Settings settings = Settings.builder().build();
+        ToLongBiFunction<ICacheKey<String>, String> weigher = getWeigher();
+        int initialKeyLength = 40;
+        int initialValueLength = 40;
+        long sizeForOneInitialEntry = weigher.applyAsLong(
+            new ICacheKey<>(generateRandomString(initialKeyLength), getMockDimensions()),
+            generateRandomString(initialValueLength)
+        );
+        int maxEntries = 2000;
         try (NodeEnvironment env = newNodeEnvironment(settings)) {
             ICache<String, String> ehcacheTest = new EhcacheDiskCache.Builder<String, String>().setDiskCacheAlias("test1")
                 .setThreadPoolAlias("ehcacheTest")
                 .setStoragePath(env.nodePaths()[0].indicesPath.toString() + "/request_cache")
-                .setIsEventListenerModeSync(true)
                 .setKeyType(String.class)
                 .setValueType(String.class)
                 .setKeySerializer(new StringSerializer())
                 .setValueSerializer(new StringSerializer())
+                .setDimensionNames(List.of(dimensionName))
+                .setIsEventListenerModeSync(true) // Test fails if async; probably not all updates happen before checking stats
+                .setCacheType(CacheType.INDICES_REQUEST_CACHE)
+                .setSettings(settings)
+                .setExpireAfterAccess(TimeValue.MAX_VALUE)
+                .setMaximumWeightInBytes(maxEntries * sizeForOneInitialEntry)
+                .setRemovalListener(new MockRemovalListener<>())
+                .setWeigher(weigher)
+                .build();
+            long expectedSize = 0;
+
+            // Test CREATED case
+            int numInitialKeys = randomIntBetween(10, 100);
+            ArrayList<ICacheKey<String>> initialKeys = new ArrayList<>();
+            for (int i = 0; i < numInitialKeys; i++) {
+                ICacheKey<String> key = new ICacheKey<>(generateRandomString(initialKeyLength), getMockDimensions());
+                String value = generateRandomString(initialValueLength);
+                ehcacheTest.put(key, value);
+                initialKeys.add(key);
+                expectedSize += weigher.applyAsLong(key, value);
+                assertEquals(expectedSize, ehcacheTest.stats().getTotalStats().getSizeInBytes());
+            }
+
+            // Test UPDATED case
+            HashMap<ICacheKey<String>, String> updatedValues = new HashMap<>();
+            for (int i = 0; i < numInitialKeys * 0.5; i++) {
+                int newLengthDifference = randomIntBetween(-20, 20);
+                String newValue = generateRandomString(initialValueLength + newLengthDifference);
+                ehcacheTest.put(initialKeys.get(i), newValue);
+                updatedValues.put(initialKeys.get(i), newValue);
+                expectedSize += newLengthDifference;
+                assertEquals(expectedSize, ehcacheTest.stats().getTotalStats().getSizeInBytes());
+            }
+
+            // Test REMOVED case by removing all updated keys
+            for (int i = 0; i < numInitialKeys * 0.5; i++) {
+                ICacheKey<String> removedKey = initialKeys.get(i);
+                ehcacheTest.invalidate(removedKey);
+                expectedSize -= weigher.applyAsLong(removedKey, updatedValues.get(removedKey));
+                assertEquals(expectedSize, ehcacheTest.stats().getTotalStats().getSizeInBytes());
+            }
+
+            // Test EVICTED case by adding entries past the cap and ensuring memory size stays as what we expect
+            for (int i = 0; i < maxEntries - ehcacheTest.count(); i++) {
+                ICacheKey<String> key = new ICacheKey<>(generateRandomString(initialKeyLength), getMockDimensions());
+                String value = generateRandomString(initialValueLength);
+                ehcacheTest.put(key, value);
+            }
+            // TODO: Ehcache incorrectly evicts at 30-40% of max size. Fix this test once we figure out why.
+            // Since the EVICTED and EXPIRED cases use the same code as REMOVED, we should be ok on testing them for now.
+            // assertEquals(maxEntries * sizeForOneInitialEntry, ehcacheTest.stats().getTotalMemorySize());
+
+            ehcacheTest.close();
+        }
+    }
+
+    public void testEhcacheKeyIteratorWithRemove() throws IOException {
+        Settings settings = Settings.builder().build();
+        try (NodeEnvironment env = newNodeEnvironment(settings)) {
+            ICache<String, String> ehcacheTest = new EhcacheDiskCache.Builder<String, String>().setDiskCacheAlias("test1")
+                .setThreadPoolAlias("ehcacheTest")
+                .setStoragePath(env.nodePaths()[0].indicesPath.toString() + "/request_cache")
+                .setIsEventListenerModeSync(true)
+                .setKeySerializer(new StringSerializer())
+                .setValueSerializer(new StringSerializer())
+                .setDimensionNames(List.of(dimensionName))
                 .setCacheType(CacheType.INDICES_REQUEST_CACHE)
+                .setKeyType(String.class)
+                .setValueType(String.class)
                 .setSettings(settings)
                 .setExpireAfterAccess(TimeValue.MAX_VALUE)
                 .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES)
                 .setRemovalListener(new MockRemovalListener<>())
+                .setWeigher(getWeigher())
                 .build();
 
             int randomKeys = randomIntBetween(2, 100);
             for (int i = 0; i < randomKeys; i++) {
-                ehcacheTest.put(UUID.randomUUID().toString(), UUID.randomUUID().toString());
+                ehcacheTest.put(getICacheKey(UUID.randomUUID().toString()), UUID.randomUUID().toString());
             }
             long originalSize = ehcacheTest.count();
             assertEquals(randomKeys, originalSize);
 
             // Now try removing subset of keys and verify
-            List<String> removedKeyList = new ArrayList<>();
-            for (Iterator<String> iterator = ehcacheTest.keys().iterator(); iterator.hasNext();) {
-                String key = iterator.next();
+            List<ICacheKey<String>> removedKeyList = new ArrayList<>();
+            for (Iterator<ICacheKey<String>> iterator = ehcacheTest.keys().iterator(); iterator.hasNext();) {
+                ICacheKey<String> key = iterator.next();
                 if (randomBoolean()) {
                     removedKeyList.add(key);
                     iterator.remove();
                 }
             }
             // Verify the removed key doesn't exist anymore.
-            for (String ehcacheKey : removedKeyList) {
+            for (ICacheKey<String> ehcacheKey : removedKeyList) {
                 assertNull(ehcacheTest.get(ehcacheKey));
             }
             // Verify ehcache entry size again.
@@ -568,22 +683,24 @@ public void testInvalidateAll() throws Exception {
                 .setValueType(String.class)
                 .setKeySerializer(new StringSerializer())
                 .setValueSerializer(new StringSerializer())
+                .setDimensionNames(List.of(dimensionName))
                 .setCacheType(CacheType.INDICES_REQUEST_CACHE)
                 .setSettings(settings)
                 .setExpireAfterAccess(TimeValue.MAX_VALUE)
                 .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES)
                 .setRemovalListener(removalListener)
+                .setWeigher(getWeigher())
                 .build();
             int randomKeys = randomIntBetween(10, 100);
-            Map<String, String> keyValueMap = new HashMap<>();
+            Map<ICacheKey<String>, String> keyValueMap = new HashMap<>();
             for (int i = 0; i < randomKeys; i++) {
-                keyValueMap.put(UUID.randomUUID().toString(), UUID.randomUUID().toString());
+                keyValueMap.put(getICacheKey(UUID.randomUUID().toString()), UUID.randomUUID().toString());
             }
-            for (Map.Entry<String, String> entry : keyValueMap.entrySet()) {
+            for (Map.Entry<ICacheKey<String>, String> entry : keyValueMap.entrySet()) {
                 ehcacheTest.put(entry.getKey(), entry.getValue());
             }
             ehcacheTest.invalidateAll(); // clear all the entries.
-            for (Map.Entry<String, String> entry : keyValueMap.entrySet()) {
+            for (Map.Entry<ICacheKey<String>, String> entry : keyValueMap.entrySet()) {
                 // Verify that value is null for a removed entry.
                 assertNull(ehcacheTest.get(entry.getKey()));
             }
@@ -600,6 +717,7 @@ public void testBasicGetAndPutBytesReference() throws Exception {
                 .setStoragePath(env.nodePaths()[0].indicesPath.toString() + "/request_cache")
                 .setKeySerializer(new StringSerializer())
                 .setValueSerializer(new BytesReferenceSerializer())
+                .setDimensionNames(List.of(dimensionName))
                 .setKeyType(String.class)
                 .setValueType(BytesReference.class)
                 .setCacheType(CacheType.INDICES_REQUEST_CACHE)
@@ -607,15 +725,16 @@ public void testBasicGetAndPutBytesReference() throws Exception {
                 .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES * 20) // bigger so no evictions happen
                 .setExpireAfterAccess(TimeValue.MAX_VALUE)
                 .setRemovalListener(new MockRemovalListener<>())
+                .setWeigher((key, value) -> 1)
                 .build();
             int randomKeys = randomIntBetween(10, 100);
             int valueLength = 100;
             Random rand = Randomness.get();
-            Map<String, BytesReference> keyValueMap = new HashMap<>();
+            Map<ICacheKey<String>, BytesReference> keyValueMap = new HashMap<>();
             for (int i = 0; i < randomKeys; i++) {
                 byte[] valueBytes = new byte[valueLength];
                 rand.nextBytes(valueBytes);
-                keyValueMap.put(UUID.randomUUID().toString(), new BytesArray(valueBytes));
+                keyValueMap.put(getICacheKey(UUID.randomUUID().toString()), new BytesArray(valueBytes));
 
                 // Test a non-BytesArray implementation of BytesReference.
                 byte[] compositeBytes1 = new byte[valueLength];
@@ -623,12 +742,12 @@ public void testBasicGetAndPutBytesReference() throws Exception {
                 rand.nextBytes(compositeBytes1);
                 rand.nextBytes(compositeBytes2);
                 BytesReference composite = CompositeBytesReference.of(new BytesArray(compositeBytes1), new BytesArray(compositeBytes2));
-                keyValueMap.put(UUID.randomUUID().toString(), composite);
+                keyValueMap.put(getICacheKey(UUID.randomUUID().toString()), composite);
             }
-            for (Map.Entry<String, BytesReference> entry : keyValueMap.entrySet()) {
+            for (Map.Entry<ICacheKey<String>, BytesReference> entry : keyValueMap.entrySet()) {
                 ehCacheDiskCachingTier.put(entry.getKey(), entry.getValue());
             }
-            for (Map.Entry<String, BytesReference> entry : keyValueMap.entrySet()) {
+            for (Map.Entry<ICacheKey<String>, BytesReference> entry : keyValueMap.entrySet()) {
                 BytesReference value = ehCacheDiskCachingTier.get(entry.getKey());
                 assertEquals(entry.getValue(), value);
             }
@@ -647,29 +766,31 @@ public void testInvalidate() throws Exception {
                 .setKeySerializer(new StringSerializer())
                 .setValueSerializer(new StringSerializer())
                 .setValueType(String.class)
+                .setDimensionNames(List.of(dimensionName))
                 .setCacheType(CacheType.INDICES_REQUEST_CACHE)
                 .setSettings(settings)
                 .setExpireAfterAccess(TimeValue.MAX_VALUE)
                 .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES)
                 .setRemovalListener(removalListener)
+                .setWeigher(getWeigher())
                 .build();
             int randomKeys = randomIntBetween(10, 100);
-            Map<String, String> keyValueMap = new HashMap<>();
+            Map<ICacheKey<String>, String> keyValueMap = new HashMap<>();
             for (int i = 0; i < randomKeys; i++) {
-                keyValueMap.put(UUID.randomUUID().toString(), UUID.randomUUID().toString());
+                keyValueMap.put(getICacheKey(UUID.randomUUID().toString()), UUID.randomUUID().toString());
             }
-            for (Map.Entry<String, String> entry : keyValueMap.entrySet()) {
+            for (Map.Entry<ICacheKey<String>, String> entry : keyValueMap.entrySet()) {
                 ehcacheTest.put(entry.getKey(), entry.getValue());
             }
             assertEquals(keyValueMap.size(), ehcacheTest.count());
-            List<String> removedKeyList = new ArrayList<>();
-            for (Map.Entry<String, String> entry : keyValueMap.entrySet()) {
+            List<ICacheKey<String>> removedKeyList = new ArrayList<>();
+            for (Map.Entry<ICacheKey<String>, String> entry : keyValueMap.entrySet()) {
                 if (randomBoolean()) {
                     removedKeyList.add(entry.getKey());
                     ehcacheTest.invalidate(entry.getKey());
                 }
             }
-            for (String removedKey : removedKeyList) {
+            for (ICacheKey<String> removedKey : removedKeyList) {
                 assertNull(ehcacheTest.get(removedKey));
             }
             assertEquals(keyValueMap.size() - removedKeyList.size(), ehcacheTest.count());
@@ -677,6 +798,100 @@ public void testInvalidate() throws Exception {
         }
     }
 
+    // Modified from OpenSearchOnHeapCacheTests.java
+    public void testInvalidateWithDropDimensions() throws Exception {
+        Settings settings = Settings.builder().build();
+        List<String> dimensionNames = List.of("dim1", "dim2");
+        try (NodeEnvironment env = newNodeEnvironment(settings)) {
+            ICache<String, String> ehCacheDiskCachingTier = new EhcacheDiskCache.Builder<String, String>().setThreadPoolAlias("ehcacheTest")
+                .setStoragePath(env.nodePaths()[0].indicesPath.toString() + "/request_cache")
+                .setKeySerializer(new StringSerializer())
+                .setValueSerializer(new StringSerializer())
+                .setIsEventListenerModeSync(true)
+                .setDimensionNames(dimensionNames)
+                .setKeyType(String.class)
+                .setValueType(String.class)
+                .setCacheType(CacheType.INDICES_REQUEST_CACHE)
+                .setSettings(settings)
+                .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES * 20) // bigger so no evictions happen
+                .setExpireAfterAccess(TimeValue.MAX_VALUE)
+                .setRemovalListener(new MockRemovalListener<>())
+                .setWeigher((key, value) -> 1)
+                .build();
+
+            List<ICacheKey<String>> keysAdded = new ArrayList<>();
+
+            for (int i = 0; i < 20; i++) {
+                ICacheKey<String> key = new ICacheKey<>(UUID.randomUUID().toString(), getRandomDimensions(dimensionNames));
+                keysAdded.add(key);
+                ehCacheDiskCachingTier.put(key, UUID.randomUUID().toString());
+            }
+
+            ICacheKey<String> keyToDrop = keysAdded.get(0);
+
+            String[] levels = dimensionNames.toArray(new String[0]);
+            ImmutableCacheStats snapshot = ehCacheDiskCachingTier.stats(levels).getStatsForDimensionValues(keyToDrop.dimensions);
+            assertNotNull(snapshot);
+
+            keyToDrop.setDropStatsForDimensions(true);
+            ehCacheDiskCachingTier.invalidate(keyToDrop);
+
+            // Now assert the stats are gone for any key that has this combination of dimensions, but still there otherwise
+            for (ICacheKey<String> keyAdded : keysAdded) {
+                snapshot = ehCacheDiskCachingTier.stats(levels).getStatsForDimensionValues(keyAdded.dimensions);
+                if (keyAdded.dimensions.equals(keyToDrop.dimensions)) {
+                    assertNull(snapshot);
+                } else {
+                    assertNotNull(snapshot);
+                }
+            }
+
+            ehCacheDiskCachingTier.close();
+        }
+    }
+
+    public void testStatsTrackingDisabled() throws Exception {
+        Settings settings = Settings.builder().build();
+        MockRemovalListener<String, String> removalListener = new MockRemovalListener<>();
+        ToLongBiFunction<ICacheKey<String>, String> weigher = getWeigher();
+        try (NodeEnvironment env = newNodeEnvironment(settings)) {
+            ICache<String, String> ehcacheTest = new EhcacheDiskCache.Builder<String, String>().setThreadPoolAlias("ehcacheTest")
+                .setStoragePath(env.nodePaths()[0].indicesPath.toString() + "/request_cache")
+                .setIsEventListenerModeSync(true)
+                .setKeyType(String.class)
+                .setValueType(String.class)
+                .setKeySerializer(new StringSerializer())
+                .setValueSerializer(new StringSerializer())
+                .setDimensionNames(List.of(dimensionName))
+                .setCacheType(CacheType.INDICES_REQUEST_CACHE)
+                .setSettings(settings)
+                .setExpireAfterAccess(TimeValue.MAX_VALUE)
+                .setMaximumWeightInBytes(CACHE_SIZE_IN_BYTES)
+                .setRemovalListener(removalListener)
+                .setWeigher(weigher)
+                .setStatsTrackingEnabled(false)
+                .build();
+            int randomKeys = randomIntBetween(10, 100);
+            for (int i = 0; i < randomKeys; i++) {
+                ICacheKey<String> iCacheKey = getICacheKey(UUID.randomUUID().toString());
+                ehcacheTest.put(iCacheKey, UUID.randomUUID().toString());
+                assertEquals(0, ehcacheTest.count()); // Expect count of 0 if NoopCacheStatsHolder is used
+                assertEquals(new ImmutableCacheStats(0, 0, 0, 0, 0), ehcacheTest.stats().getTotalStats());
+            }
+            ehcacheTest.close();
+        }
+    }
+
+    private List<String> getRandomDimensions(List<String> dimensionNames) {
+        Random rand = Randomness.get();
+        int bound = 3;
+        List<String> result = new ArrayList<>();
+        for (String dimName : dimensionNames) {
+            result.add(String.valueOf(rand.nextInt(bound)));
+        }
+        return result;
+    }
+
     private static String generateRandomString(int length) {
         String characters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
         StringBuilder randomString = new StringBuilder(length);
@@ -689,12 +904,34 @@ private static String generateRandomString(int length) {
         return randomString.toString();
     }
 
-    static class MockRemovalListener<K, V> implements RemovalListener<K, V> {
+    private List<String> getMockDimensions() {
+        return List.of("0");
+    }
+
+    private ICacheKey<String> getICacheKey(String key) {
+        return new ICacheKey<>(key, getMockDimensions());
+    }
+
+    private ToLongBiFunction<ICacheKey<String>, String> getWeigher() {
+        return (iCacheKey, value) -> {
+            // Size consumed by key
+            long totalSize = iCacheKey.key.length();
+            for (String dim : iCacheKey.dimensions) {
+                totalSize += dim.length();
+            }
+            totalSize += 10; // The ICacheKeySerializer writes 2 VInts to record array lengths, which can be 1-5 bytes each
+            // Size consumed by value
+            totalSize += value.length();
+            return totalSize;
+        };
+    }
+
+    static class MockRemovalListener<K, V> implements RemovalListener<ICacheKey<K>, V> {
 
         CounterMetric evictionMetric = new CounterMetric();
 
         @Override
-        public void onRemoval(RemovalNotification<K, V> notification) {
+        public void onRemoval(RemovalNotification<ICacheKey<K>, V> notification) {
             evictionMetric.inc();
         }
     }
diff --git a/plugins/crypto-kms/licenses/commons-codec-1.15.jar.sha1 b/plugins/crypto-kms/licenses/commons-codec-1.15.jar.sha1
deleted file mode 100644
index 62d99837b87e1..0000000000000
--- a/plugins/crypto-kms/licenses/commons-codec-1.15.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-49d94806b6e3dc933dacbd8acb0fdbab8ebd1e5d
\ No newline at end of file
diff --git a/plugins/crypto-kms/licenses/commons-codec-1.16.1.jar.sha1 b/plugins/crypto-kms/licenses/commons-codec-1.16.1.jar.sha1
new file mode 100644
index 0000000000000..6b8803089c6d7
--- /dev/null
+++ b/plugins/crypto-kms/licenses/commons-codec-1.16.1.jar.sha1
@@ -0,0 +1 @@
+47bd4d333fba53406f6c6c51884ddbca435c8862
\ No newline at end of file
diff --git a/plugins/discovery-azure-classic/build.gradle b/plugins/discovery-azure-classic/build.gradle
index c3d70e9c64968..7f34cec94499c 100644
--- a/plugins/discovery-azure-classic/build.gradle
+++ b/plugins/discovery-azure-classic/build.gradle
@@ -53,7 +53,7 @@ dependencies {
   api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
   api "commons-codec:commons-codec:${versions.commonscodec}"
   api "commons-lang:commons-lang:2.6"
-  api "commons-io:commons-io:2.15.1"
+  api "commons-io:commons-io:${versions.commonsio}"
   api 'javax.mail:mail:1.4.7'
   api 'javax.inject:javax.inject:1'
   api "com.sun.jersey:jersey-client:${versions.jersey}"
diff --git a/plugins/discovery-azure-classic/licenses/commons-codec-1.15.jar.sha1 b/plugins/discovery-azure-classic/licenses/commons-codec-1.15.jar.sha1
deleted file mode 100644
index 62d99837b87e1..0000000000000
--- a/plugins/discovery-azure-classic/licenses/commons-codec-1.15.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-49d94806b6e3dc933dacbd8acb0fdbab8ebd1e5d
\ No newline at end of file
diff --git a/plugins/discovery-azure-classic/licenses/commons-codec-1.16.1.jar.sha1 b/plugins/discovery-azure-classic/licenses/commons-codec-1.16.1.jar.sha1
new file mode 100644
index 0000000000000..6b8803089c6d7
--- /dev/null
+++ b/plugins/discovery-azure-classic/licenses/commons-codec-1.16.1.jar.sha1
@@ -0,0 +1 @@
+47bd4d333fba53406f6c6c51884ddbca435c8862
\ No newline at end of file
diff --git a/plugins/discovery-azure-classic/licenses/commons-io-2.15.1.jar.sha1 b/plugins/discovery-azure-classic/licenses/commons-io-2.15.1.jar.sha1
deleted file mode 100644
index 47c5d13812a36..0000000000000
--- a/plugins/discovery-azure-classic/licenses/commons-io-2.15.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f11560da189ab563a5c8e351941415430e9304ea
\ No newline at end of file
diff --git a/plugins/discovery-azure-classic/licenses/commons-io-2.16.0.jar.sha1 b/plugins/discovery-azure-classic/licenses/commons-io-2.16.0.jar.sha1
new file mode 100644
index 0000000000000..6a7b638719fa3
--- /dev/null
+++ b/plugins/discovery-azure-classic/licenses/commons-io-2.16.0.jar.sha1
@@ -0,0 +1 @@
+27875a7935f1ddcc13267eb6fae1f719e0409572
\ No newline at end of file
diff --git a/plugins/discovery-ec2/licenses/commons-codec-1.15.jar.sha1 b/plugins/discovery-ec2/licenses/commons-codec-1.15.jar.sha1
deleted file mode 100644
index 62d99837b87e1..0000000000000
--- a/plugins/discovery-ec2/licenses/commons-codec-1.15.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-49d94806b6e3dc933dacbd8acb0fdbab8ebd1e5d
\ No newline at end of file
diff --git a/plugins/discovery-ec2/licenses/commons-codec-1.16.1.jar.sha1 b/plugins/discovery-ec2/licenses/commons-codec-1.16.1.jar.sha1
new file mode 100644
index 0000000000000..6b8803089c6d7
--- /dev/null
+++ b/plugins/discovery-ec2/licenses/commons-codec-1.16.1.jar.sha1
@@ -0,0 +1 @@
+47bd4d333fba53406f6c6c51884ddbca435c8862
\ No newline at end of file
diff --git a/plugins/discovery-gce/build.gradle b/plugins/discovery-gce/build.gradle
index 85efcc43fd65a..80aae03bc0332 100644
--- a/plugins/discovery-gce/build.gradle
+++ b/plugins/discovery-gce/build.gradle
@@ -17,22 +17,23 @@ opensearchplugin {
   classname 'org.opensearch.plugin.discovery.gce.GceDiscoveryPlugin'
 }
 
-versions << [
-  'google': '1.23.0'
-]
-
 dependencies {
-  api "com.google.apis:google-api-services-compute:v1-rev160-${versions.google}"
-  api "com.google.api-client:google-api-client:${versions.google}"
+  api "com.google.apis:google-api-services-compute:v1-rev20240407-2.0.0"
+  api "com.google.api-client:google-api-client:1.35.2"
   api "com.google.oauth-client:google-oauth-client:1.35.0"
-  api "com.google.http-client:google-http-client:${versions.google}"
-  api "com.google.http-client:google-http-client-jackson2:${versions.google}"
+  api "com.google.http-client:google-http-client:${versions.google_http_client}"
+  api "com.google.http-client:google-http-client-gson:${versions.google_http_client}"
+  api "com.google.http-client:google-http-client-jackson2:${versions.google_http_client}"
   api 'com.google.code.findbugs:jsr305:3.0.2'
   api "org.apache.httpcomponents:httpclient:${versions.httpclient}"
   api "org.apache.httpcomponents:httpcore:${versions.httpcore}"
   api "commons-logging:commons-logging:${versions.commonslogging}"
   api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
   api "commons-codec:commons-codec:${versions.commonscodec}"
+  api 'io.grpc:grpc-api:1.57.2'
+  api 'io.opencensus:opencensus-api:0.31.1'
+  api 'io.opencensus:opencensus-contrib-http-util:0.31.1'
+  runtimeOnly "com.google.guava:guava:${versions.guava}"
 }
 
 restResources {
@@ -43,6 +44,7 @@ restResources {
 
 tasks.named("dependencyLicenses").configure {
   mapping from: /google-.*/, to: 'google'
+  mapping from: /opencensus.*/, to: 'opencensus'
 }
 
 check {
@@ -55,26 +57,36 @@ test {
   systemProperty 'tests.artifact', project.name
 }
 
-thirdPartyAudit.ignoreMissingClasses(
-  // classes are missing
-  'javax.jms.Message',
-  'javax.servlet.ServletContextEvent',
-  'javax.servlet.ServletContextListener',
-  'org.apache.avalon.framework.logger.Logger',
-  'org.apache.log.Hierarchy',
-  'org.apache.log.Logger',
-  'com.google.api.client.json.gson.GsonFactory',
-  'com.google.common.base.Preconditions',
-  'com.google.common.base.Splitter',
-  'com.google.common.cache.CacheBuilder',
-  'com.google.common.cache.CacheLoader',
-  'com.google.common.cache.LoadingCache',
-  'com.google.common.collect.ImmutableMap',
-  'com.google.common.collect.ImmutableMap$Builder',
-  'com.google.common.collect.ImmutableSet',
-  'com.google.common.collect.Lists',
-  'com.google.common.collect.Multiset',
-  'com.google.common.collect.SortedMultiset',
-  'com.google.common.collect.TreeMultiset',
-  'com.google.common.io.BaseEncoding',
-)
+thirdPartyAudit {
+  ignoreViolations(
+    // uses internal java api: sun.misc.Unsafe
+    'com.google.common.cache.Striped64',
+    'com.google.common.cache.Striped64$1',
+    'com.google.common.cache.Striped64$Cell',
+    'com.google.common.hash.Striped64',
+    'com.google.common.hash.Striped64$1',
+    'com.google.common.hash.Striped64$Cell',
+    'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray',
+    'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$1',
+    'com.google.common.hash.LittleEndianByteArray$UnsafeByteArray$2',
+    'com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper',
+    'com.google.common.util.concurrent.AbstractFuture$UnsafeAtomicHelper$1',
+    'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator',
+    'com.google.common.primitives.UnsignedBytes$LexicographicalComparatorHolder$UnsafeComparator$1',
+  )
+
+  ignoreMissingClasses(
+    'com.google.api.client.http.apache.v2.ApacheHttpTransport',
+    'com.google.common.util.concurrent.internal.InternalFutureFailureAccess',
+    'com.google.common.util.concurrent.internal.InternalFutures',
+    'com.google.gson.stream.JsonReader',
+    'com.google.gson.stream.JsonToken',
+    'com.google.gson.stream.JsonWriter',
+    'javax.jms.Message',
+    'javax.servlet.ServletContextEvent',
+    'javax.servlet.ServletContextListener',
+    'org.apache.avalon.framework.logger.Logger',
+    'org.apache.log.Hierarchy',
+    'org.apache.log.Logger'
+  )
+}
diff --git a/plugins/discovery-gce/licenses/commons-codec-1.15.jar.sha1 b/plugins/discovery-gce/licenses/commons-codec-1.15.jar.sha1
deleted file mode 100644
index 62d99837b87e1..0000000000000
--- a/plugins/discovery-gce/licenses/commons-codec-1.15.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-49d94806b6e3dc933dacbd8acb0fdbab8ebd1e5d
\ No newline at end of file
diff --git a/plugins/discovery-gce/licenses/commons-codec-1.16.1.jar.sha1 b/plugins/discovery-gce/licenses/commons-codec-1.16.1.jar.sha1
new file mode 100644
index 0000000000000..6b8803089c6d7
--- /dev/null
+++ b/plugins/discovery-gce/licenses/commons-codec-1.16.1.jar.sha1
@@ -0,0 +1 @@
+47bd4d333fba53406f6c6c51884ddbca435c8862
\ No newline at end of file
diff --git a/plugins/discovery-gce/licenses/google-api-client-1.23.0.jar.sha1 b/plugins/discovery-gce/licenses/google-api-client-1.23.0.jar.sha1
deleted file mode 100644
index 0c35d8e08b91f..0000000000000
--- a/plugins/discovery-gce/licenses/google-api-client-1.23.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-522ea860eb48dee71dfe2c61a1fd09663539f556
\ No newline at end of file
diff --git a/plugins/discovery-gce/licenses/google-api-client-1.35.2.jar.sha1 b/plugins/discovery-gce/licenses/google-api-client-1.35.2.jar.sha1
new file mode 100644
index 0000000000000..47245f9429e7d
--- /dev/null
+++ b/plugins/discovery-gce/licenses/google-api-client-1.35.2.jar.sha1
@@ -0,0 +1 @@
+2d737980e34c674da4ff0ae124b80caefdc7198a
\ No newline at end of file
diff --git a/plugins/discovery-gce/licenses/google-api-services-compute-v1-rev160-1.23.0.jar.sha1 b/plugins/discovery-gce/licenses/google-api-services-compute-v1-rev160-1.23.0.jar.sha1
deleted file mode 100644
index 17219dfe7ecc9..0000000000000
--- a/plugins/discovery-gce/licenses/google-api-services-compute-v1-rev160-1.23.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-004169bfe1cf0e8b2013c9c479e43b731958bc64
\ No newline at end of file
diff --git a/plugins/discovery-gce/licenses/google-api-services-compute-v1-rev20240407-2.0.0.jar.sha1 b/plugins/discovery-gce/licenses/google-api-services-compute-v1-rev20240407-2.0.0.jar.sha1
new file mode 100644
index 0000000000000..834d718641a51
--- /dev/null
+++ b/plugins/discovery-gce/licenses/google-api-services-compute-v1-rev20240407-2.0.0.jar.sha1
@@ -0,0 +1 @@
+edf93bc92c9b87fee51aa6c3545b565e58075c05
\ No newline at end of file
diff --git a/plugins/discovery-gce/licenses/google-http-client-1.23.0.jar.sha1 b/plugins/discovery-gce/licenses/google-http-client-1.23.0.jar.sha1
deleted file mode 100644
index 5526275d5a15f..0000000000000
--- a/plugins/discovery-gce/licenses/google-http-client-1.23.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-8e86c84ff3c98eca6423e97780325b299133d858
\ No newline at end of file
diff --git a/plugins/discovery-gce/licenses/google-http-client-1.44.1.jar.sha1 b/plugins/discovery-gce/licenses/google-http-client-1.44.1.jar.sha1
new file mode 100644
index 0000000000000..501f268254fbc
--- /dev/null
+++ b/plugins/discovery-gce/licenses/google-http-client-1.44.1.jar.sha1
@@ -0,0 +1 @@
+d8956bacb8a4011365fa15a690482c49a70c78c5
\ No newline at end of file
diff --git a/plugins/discovery-gce/licenses/google-http-client-gson-1.44.1.jar.sha1 b/plugins/discovery-gce/licenses/google-http-client-gson-1.44.1.jar.sha1
new file mode 100644
index 0000000000000..90ddf3ddc5ee6
--- /dev/null
+++ b/plugins/discovery-gce/licenses/google-http-client-gson-1.44.1.jar.sha1
@@ -0,0 +1 @@
+f3b8967c6f7078da6380687859d0873105f84d39
\ No newline at end of file
diff --git a/plugins/discovery-gce/licenses/google-http-client-jackson2-1.23.0.jar.sha1 b/plugins/discovery-gce/licenses/google-http-client-jackson2-1.23.0.jar.sha1
deleted file mode 100644
index 510856a517f04..0000000000000
--- a/plugins/discovery-gce/licenses/google-http-client-jackson2-1.23.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-fd6761f4046a8cb0455e6fa5f58e12b061e9826e
\ No newline at end of file
diff --git a/plugins/discovery-gce/licenses/google-http-client-jackson2-1.44.1.jar.sha1 b/plugins/discovery-gce/licenses/google-http-client-jackson2-1.44.1.jar.sha1
new file mode 100644
index 0000000000000..4472ffbbebe1c
--- /dev/null
+++ b/plugins/discovery-gce/licenses/google-http-client-jackson2-1.44.1.jar.sha1
@@ -0,0 +1 @@
+3f1947de0fd9eb250af16abe6103c11e68d11635
\ No newline at end of file
diff --git a/plugins/discovery-gce/licenses/grpc-api-1.57.2.jar.sha1 b/plugins/discovery-gce/licenses/grpc-api-1.57.2.jar.sha1
new file mode 100644
index 0000000000000..8b320fdd2f9cc
--- /dev/null
+++ b/plugins/discovery-gce/licenses/grpc-api-1.57.2.jar.sha1
@@ -0,0 +1 @@
+c71a006b81ddae7bc4b7cb1d2da78c1b173761f4
\ No newline at end of file
diff --git a/plugins/discovery-gce/licenses/grpc-api-LICENSE.txt b/plugins/discovery-gce/licenses/grpc-api-LICENSE.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/plugins/discovery-gce/licenses/grpc-api-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/plugins/discovery-gce/licenses/grpc-api-NOTICE.txt b/plugins/discovery-gce/licenses/grpc-api-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/plugins/discovery-gce/licenses/guava-32.1.1-jre.jar.sha1 b/plugins/discovery-gce/licenses/guava-32.1.1-jre.jar.sha1
new file mode 100644
index 0000000000000..0d791b5d3f55b
--- /dev/null
+++ b/plugins/discovery-gce/licenses/guava-32.1.1-jre.jar.sha1
@@ -0,0 +1 @@
+ad575652d84153075dd41ec6177ccb15251262b2
\ No newline at end of file
diff --git a/plugins/discovery-gce/licenses/guava-LICENSE.txt b/plugins/discovery-gce/licenses/guava-LICENSE.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/plugins/discovery-gce/licenses/guava-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/plugins/discovery-gce/licenses/guava-NOTICE.txt b/plugins/discovery-gce/licenses/guava-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/plugins/discovery-gce/licenses/opencensus-LICENSE.txt b/plugins/discovery-gce/licenses/opencensus-LICENSE.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/plugins/discovery-gce/licenses/opencensus-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/plugins/discovery-gce/licenses/opencensus-NOTICE.txt b/plugins/discovery-gce/licenses/opencensus-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/plugins/discovery-gce/licenses/opencensus-api-0.31.1.jar.sha1 b/plugins/discovery-gce/licenses/opencensus-api-0.31.1.jar.sha1
new file mode 100644
index 0000000000000..03760848f76ef
--- /dev/null
+++ b/plugins/discovery-gce/licenses/opencensus-api-0.31.1.jar.sha1
@@ -0,0 +1 @@
+66a60c7201c2b8b20ce495f0295b32bb0ccbbc57
\ No newline at end of file
diff --git a/plugins/discovery-gce/licenses/opencensus-contrib-http-util-0.31.1.jar.sha1 b/plugins/discovery-gce/licenses/opencensus-contrib-http-util-0.31.1.jar.sha1
new file mode 100644
index 0000000000000..4e123da3ab45f
--- /dev/null
+++ b/plugins/discovery-gce/licenses/opencensus-contrib-http-util-0.31.1.jar.sha1
@@ -0,0 +1 @@
+3c13fc5715231fadb16a9b74a44d9d59c460cfa8
\ No newline at end of file
diff --git a/plugins/identity-shiro/licenses/bcprov-jdk18on-1.77.jar.sha1 b/plugins/identity-shiro/licenses/bcprov-jdk18on-1.77.jar.sha1
deleted file mode 100644
index 3e780df9559a9..0000000000000
--- a/plugins/identity-shiro/licenses/bcprov-jdk18on-1.77.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-2cc971b6c20949c1ff98d1a4bc741ee848a09523
\ No newline at end of file
diff --git a/plugins/identity-shiro/licenses/bcprov-jdk18on-1.78.jar.sha1 b/plugins/identity-shiro/licenses/bcprov-jdk18on-1.78.jar.sha1
new file mode 100644
index 0000000000000..47fb5fd5e5f5d
--- /dev/null
+++ b/plugins/identity-shiro/licenses/bcprov-jdk18on-1.78.jar.sha1
@@ -0,0 +1 @@
+619aafb92dc0b4c6cc4cf86c487ca48ee2d67a8e
\ No newline at end of file
diff --git a/plugins/ingest-attachment/build.gradle b/plugins/ingest-attachment/build.gradle
index 6da34c4c9caf2..4749aa911886d 100644
--- a/plugins/ingest-attachment/build.gradle
+++ b/plugins/ingest-attachment/build.gradle
@@ -38,10 +38,10 @@ opensearchplugin {
 }
 
 versions << [
-  'tika'  : '2.6.0',
-  'pdfbox': '2.0.27',
-  'poi'   : '5.2.3',
-  'mime4j': '0.8.8'
+  'tika'  : '2.9.2',
+  'pdfbox': '2.0.31',
+  'poi'   : '5.2.5',
+  'mime4j': '0.8.11'
 ]
 
 dependencies {
@@ -50,6 +50,16 @@ dependencies {
   api "org.apache.tika:tika-parsers:${versions.tika}"
   // Required for the various document parsers
   api "org.apache.tika:tika-parsers-standard-package:${versions.tika}"
+  api "org.apache.tika:tika-parser-apple-module:${versions.tika}"
+  api "org.apache.tika:tika-parser-html-module:${versions.tika}"
+  api "org.apache.tika:tika-parser-microsoft-module:${versions.tika}"
+  api "org.apache.tika:tika-parser-miscoffice-module:${versions.tika}"
+  api "org.apache.tika:tika-parser-pdf-module:${versions.tika}"
+  api "org.apache.tika:tika-parser-text-module:${versions.tika}"
+  api "org.apache.tika:tika-parser-xml-module:${versions.tika}"
+  // Utilities consumed by document parsers
+  api "org.apache.tika:tika-parser-xmp-commons:${versions.tika}"
+  api "org.apache.tika:tika-parser-zip-commons:${versions.tika}"
   // Required for language detection
   api "org.apache.tika:tika-langdetect-optimaize:${versions.tika}"
   // Optimaize libraries/dependencies
@@ -57,7 +67,7 @@ dependencies {
   runtimeOnly "com.google.guava:guava:${versions.guava}"
   // Other dependencies
   api 'org.tukaani:xz:1.9'
-  api 'commons-io:commons-io:2.15.1'
+  api "commons-io:commons-io:${versions.commonsio}"
   api "org.slf4j:slf4j-api:${versions.slf4j}"
 
   // character set detection
diff --git a/plugins/ingest-attachment/licenses/apache-mime4j-core-0.8.11.jar.sha1 b/plugins/ingest-attachment/licenses/apache-mime4j-core-0.8.11.jar.sha1
new file mode 100644
index 0000000000000..82d9bf2617ce6
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/apache-mime4j-core-0.8.11.jar.sha1
@@ -0,0 +1 @@
+6d1eb5f7b84eaa9d38fca13b761f01c693aef3da
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/apache-mime4j-core-0.8.8.jar.sha1 b/plugins/ingest-attachment/licenses/apache-mime4j-core-0.8.8.jar.sha1
deleted file mode 100644
index 77c36691d36b5..0000000000000
--- a/plugins/ingest-attachment/licenses/apache-mime4j-core-0.8.8.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-7330de23c52f71617cbec7f1d2760dae32e687cd
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/apache-mime4j-dom-0.8.11.jar.sha1 b/plugins/ingest-attachment/licenses/apache-mime4j-dom-0.8.11.jar.sha1
new file mode 100644
index 0000000000000..7a494aba6a231
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/apache-mime4j-dom-0.8.11.jar.sha1
@@ -0,0 +1 @@
+f0d42ab9a5832b5f5d05afc004b31245b838e0fc
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/apache-mime4j-dom-0.8.8.jar.sha1 b/plugins/ingest-attachment/licenses/apache-mime4j-dom-0.8.8.jar.sha1
deleted file mode 100644
index fb9c5fed27162..0000000000000
--- a/plugins/ingest-attachment/licenses/apache-mime4j-dom-0.8.8.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-e76715563a6bd150f84ccb0adb920aec8faf4779
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/bcmail-jdk18on-1.77.jar.sha1 b/plugins/ingest-attachment/licenses/bcmail-jdk18on-1.77.jar.sha1
deleted file mode 100644
index f71659316b8cd..0000000000000
--- a/plugins/ingest-attachment/licenses/bcmail-jdk18on-1.77.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f2bb8aa55dc901ee8b8aae7d1007c03592d65e03
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/bcmail-jdk18on-1.78.jar.sha1 b/plugins/ingest-attachment/licenses/bcmail-jdk18on-1.78.jar.sha1
new file mode 100644
index 0000000000000..eb7e650306f73
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/bcmail-jdk18on-1.78.jar.sha1
@@ -0,0 +1 @@
+d26f5514b8c54f2878f8d49e0bc8e2acaab3c8bd
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/bcpkix-jdk18on-1.77.jar.sha1 b/plugins/ingest-attachment/licenses/bcpkix-jdk18on-1.77.jar.sha1
deleted file mode 100644
index 05a8b2d5729bd..0000000000000
--- a/plugins/ingest-attachment/licenses/bcpkix-jdk18on-1.77.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-ed953791ba0229747dd0fd9911e3d76a462acfd3
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/bcpkix-jdk18on-1.78.jar.sha1 b/plugins/ingest-attachment/licenses/bcpkix-jdk18on-1.78.jar.sha1
new file mode 100644
index 0000000000000..385a9d930eede
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/bcpkix-jdk18on-1.78.jar.sha1
@@ -0,0 +1 @@
+dd61bcdb87678451dd42d42e267979bd4b4451a1
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/bcprov-jdk18on-1.77.jar.sha1 b/plugins/ingest-attachment/licenses/bcprov-jdk18on-1.77.jar.sha1
deleted file mode 100644
index 3e780df9559a9..0000000000000
--- a/plugins/ingest-attachment/licenses/bcprov-jdk18on-1.77.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-2cc971b6c20949c1ff98d1a4bc741ee848a09523
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/bcprov-jdk18on-1.78.jar.sha1 b/plugins/ingest-attachment/licenses/bcprov-jdk18on-1.78.jar.sha1
new file mode 100644
index 0000000000000..47fb5fd5e5f5d
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/bcprov-jdk18on-1.78.jar.sha1
@@ -0,0 +1 @@
+619aafb92dc0b4c6cc4cf86c487ca48ee2d67a8e
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/commons-codec-1.15.jar.sha1 b/plugins/ingest-attachment/licenses/commons-codec-1.15.jar.sha1
deleted file mode 100644
index 62d99837b87e1..0000000000000
--- a/plugins/ingest-attachment/licenses/commons-codec-1.15.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-49d94806b6e3dc933dacbd8acb0fdbab8ebd1e5d
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/commons-codec-1.16.1.jar.sha1 b/plugins/ingest-attachment/licenses/commons-codec-1.16.1.jar.sha1
new file mode 100644
index 0000000000000..6b8803089c6d7
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/commons-codec-1.16.1.jar.sha1
@@ -0,0 +1 @@
+47bd4d333fba53406f6c6c51884ddbca435c8862
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/commons-compress-1.24.0.jar.sha1 b/plugins/ingest-attachment/licenses/commons-compress-1.24.0.jar.sha1
deleted file mode 100644
index 23999d1bfbde4..0000000000000
--- a/plugins/ingest-attachment/licenses/commons-compress-1.24.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b4b1b5a3d9573b2970fddab236102c0a4d27d35e
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/commons-compress-1.26.1.jar.sha1 b/plugins/ingest-attachment/licenses/commons-compress-1.26.1.jar.sha1
new file mode 100644
index 0000000000000..912bda85de18a
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/commons-compress-1.26.1.jar.sha1
@@ -0,0 +1 @@
+44331c1130c370e726a2e1a3e6fba6d2558ef04a
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/commons-io-2.15.1.jar.sha1 b/plugins/ingest-attachment/licenses/commons-io-2.15.1.jar.sha1
deleted file mode 100644
index 47c5d13812a36..0000000000000
--- a/plugins/ingest-attachment/licenses/commons-io-2.15.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f11560da189ab563a5c8e351941415430e9304ea
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/commons-io-2.16.0.jar.sha1 b/plugins/ingest-attachment/licenses/commons-io-2.16.0.jar.sha1
new file mode 100644
index 0000000000000..6a7b638719fa3
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/commons-io-2.16.0.jar.sha1
@@ -0,0 +1 @@
+27875a7935f1ddcc13267eb6fae1f719e0409572
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/commons-lang3-3.13.0.jar.sha1 b/plugins/ingest-attachment/licenses/commons-lang3-3.13.0.jar.sha1
deleted file mode 100644
index d0c2f2486ee1f..0000000000000
--- a/plugins/ingest-attachment/licenses/commons-lang3-3.13.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b7263237aa89c1f99b327197c41d0669707a462e
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/commons-lang3-3.14.0.jar.sha1 b/plugins/ingest-attachment/licenses/commons-lang3-3.14.0.jar.sha1
new file mode 100644
index 0000000000000..d783e07e40902
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/commons-lang3-3.14.0.jar.sha1
@@ -0,0 +1 @@
+1ed471194b02f2c6cb734a0cd6f6f107c673afae
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/fontbox-2.0.27.jar.sha1 b/plugins/ingest-attachment/licenses/fontbox-2.0.27.jar.sha1
deleted file mode 100644
index d578dffbfa3f6..0000000000000
--- a/plugins/ingest-attachment/licenses/fontbox-2.0.27.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d08c064d18b2b149da937d15c0d1708cba03f29d
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/fontbox-2.0.31.jar.sha1 b/plugins/ingest-attachment/licenses/fontbox-2.0.31.jar.sha1
new file mode 100644
index 0000000000000..d45d45a66e072
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/fontbox-2.0.31.jar.sha1
@@ -0,0 +1 @@
+96999ecdb7324bf718b88724818fa62f81286c36
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/pdfbox-2.0.27.jar.sha1 b/plugins/ingest-attachment/licenses/pdfbox-2.0.27.jar.sha1
deleted file mode 100644
index 4f670b7f95e8c..0000000000000
--- a/plugins/ingest-attachment/licenses/pdfbox-2.0.27.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-416a9dfce3714116bfdf793b15368df04266845f
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/pdfbox-2.0.31.jar.sha1 b/plugins/ingest-attachment/licenses/pdfbox-2.0.31.jar.sha1
new file mode 100644
index 0000000000000..fa256ed9a65d2
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/pdfbox-2.0.31.jar.sha1
@@ -0,0 +1 @@
+29b25053099bc30784a766ccb821417e06f4b8a1
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/poi-5.2.3.jar.sha1 b/plugins/ingest-attachment/licenses/poi-5.2.3.jar.sha1
deleted file mode 100644
index 3d8b3daf606ad..0000000000000
--- a/plugins/ingest-attachment/licenses/poi-5.2.3.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-2fb22ae74ad5aea6af1a9c64b9542f2ccf348604
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/poi-5.2.5.jar.sha1 b/plugins/ingest-attachment/licenses/poi-5.2.5.jar.sha1
new file mode 100644
index 0000000000000..0eca17726eb0b
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/poi-5.2.5.jar.sha1
@@ -0,0 +1 @@
+7e00f6b2f76375fe89022d5a7db8acb71cbd55f5
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/poi-ooxml-5.2.3.jar.sha1 b/plugins/ingest-attachment/licenses/poi-ooxml-5.2.3.jar.sha1
deleted file mode 100644
index 8371593cf0841..0000000000000
--- a/plugins/ingest-attachment/licenses/poi-ooxml-5.2.3.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-02efd11c940adb18c03eb9ce7ad88fc40ee6a196
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/poi-ooxml-5.2.5.jar.sha1 b/plugins/ingest-attachment/licenses/poi-ooxml-5.2.5.jar.sha1
new file mode 100644
index 0000000000000..6b14be4461425
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/poi-ooxml-5.2.5.jar.sha1
@@ -0,0 +1 @@
+df9f2c52371eeba24db8ea8cafa77285c3cc0742
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/poi-ooxml-lite-5.2.3.jar.sha1 b/plugins/ingest-attachment/licenses/poi-ooxml-lite-5.2.3.jar.sha1
deleted file mode 100644
index 5c6365876b7be..0000000000000
--- a/plugins/ingest-attachment/licenses/poi-ooxml-lite-5.2.3.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-db113c8e9051b0ff967f4911fa20336c8325a7c5
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/poi-ooxml-lite-5.2.5.jar.sha1 b/plugins/ingest-attachment/licenses/poi-ooxml-lite-5.2.5.jar.sha1
new file mode 100644
index 0000000000000..f9a473173a297
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/poi-ooxml-lite-5.2.5.jar.sha1
@@ -0,0 +1 @@
+eaa61452d8f0d13080fbb4757a392f09f90e4c49
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/poi-scratchpad-5.2.3.jar.sha1 b/plugins/ingest-attachment/licenses/poi-scratchpad-5.2.3.jar.sha1
deleted file mode 100644
index 3c8f92498f1a4..0000000000000
--- a/plugins/ingest-attachment/licenses/poi-scratchpad-5.2.3.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-2a7fce47e22b7fedb1b277347ff4fe36d6eda50d
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/poi-scratchpad-5.2.5.jar.sha1 b/plugins/ingest-attachment/licenses/poi-scratchpad-5.2.5.jar.sha1
new file mode 100644
index 0000000000000..68665ddafd7d8
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/poi-scratchpad-5.2.5.jar.sha1
@@ -0,0 +1 @@
+fc600cf765a49d73935a6e48a5b84f4abcdd0518
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/tika-core-2.6.0.jar.sha1 b/plugins/ingest-attachment/licenses/tika-core-2.6.0.jar.sha1
deleted file mode 100644
index c66c2f3f39401..0000000000000
--- a/plugins/ingest-attachment/licenses/tika-core-2.6.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f6ed6356dd4a9bd269d873f65494376685e6192e
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/tika-core-2.9.2.jar.sha1 b/plugins/ingest-attachment/licenses/tika-core-2.9.2.jar.sha1
new file mode 100644
index 0000000000000..80635a63d29fe
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/tika-core-2.9.2.jar.sha1
@@ -0,0 +1 @@
+796a21391780339e3d4862626339b49df170024e
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/tika-langdetect-optimaize-2.6.0.jar.sha1 b/plugins/ingest-attachment/licenses/tika-langdetect-optimaize-2.6.0.jar.sha1
deleted file mode 100644
index e7bc59bb5ae49..0000000000000
--- a/plugins/ingest-attachment/licenses/tika-langdetect-optimaize-2.6.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-72b784a7bdab0ffde005fa64d15e3f077331d6fc
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/tika-langdetect-optimaize-2.9.2.jar.sha1 b/plugins/ingest-attachment/licenses/tika-langdetect-optimaize-2.9.2.jar.sha1
new file mode 100644
index 0000000000000..a4bb6d48c6a08
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/tika-langdetect-optimaize-2.9.2.jar.sha1
@@ -0,0 +1 @@
+7a48a287e464b456a85c79f318d7bad7db201518
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/tika-parser-apple-module-2.9.2.jar.sha1 b/plugins/ingest-attachment/licenses/tika-parser-apple-module-2.9.2.jar.sha1
new file mode 100644
index 0000000000000..dbaee880d1251
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/tika-parser-apple-module-2.9.2.jar.sha1
@@ -0,0 +1 @@
+758dac27c246c51b019562bab7e266d2da6a6e01
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/tika-parser-html-module-2.9.2.jar.sha1 b/plugins/ingest-attachment/licenses/tika-parser-html-module-2.9.2.jar.sha1
new file mode 100644
index 0000000000000..b4806746301ef
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/tika-parser-html-module-2.9.2.jar.sha1
@@ -0,0 +1 @@
+47f6a4c46b92616d14e82cd7ad4d05cb43077b83
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/tika-parser-microsoft-module-2.9.2.jar.sha1 b/plugins/ingest-attachment/licenses/tika-parser-microsoft-module-2.9.2.jar.sha1
new file mode 100644
index 0000000000000..da1ae42bac652
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/tika-parser-microsoft-module-2.9.2.jar.sha1
@@ -0,0 +1 @@
+235a20823c02c699ce3d57f3d6b9550db05d91a9
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/tika-parser-miscoffice-module-2.9.2.jar.sha1 b/plugins/ingest-attachment/licenses/tika-parser-miscoffice-module-2.9.2.jar.sha1
new file mode 100644
index 0000000000000..7ceed9e1643b8
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/tika-parser-miscoffice-module-2.9.2.jar.sha1
@@ -0,0 +1 @@
+7688a4220d07c32b505230479f957cd495c0bef2
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/tika-parser-pdf-module-2.9.2.jar.sha1 b/plugins/ingest-attachment/licenses/tika-parser-pdf-module-2.9.2.jar.sha1
new file mode 100644
index 0000000000000..e780c1b92d525
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/tika-parser-pdf-module-2.9.2.jar.sha1
@@ -0,0 +1 @@
+4d0f0e3f6eff184040402094f4fabbb3c5c7d09f
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/tika-parser-text-module-2.9.2.jar.sha1 b/plugins/ingest-attachment/licenses/tika-parser-text-module-2.9.2.jar.sha1
new file mode 100644
index 0000000000000..6e56fcffc5f88
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/tika-parser-text-module-2.9.2.jar.sha1
@@ -0,0 +1 @@
+b3a93e538ba6cb4066aba96d629febf181ec9f92
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/tika-parser-xml-module-2.9.2.jar.sha1 b/plugins/ingest-attachment/licenses/tika-parser-xml-module-2.9.2.jar.sha1
new file mode 100644
index 0000000000000..27062077b92bf
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/tika-parser-xml-module-2.9.2.jar.sha1
@@ -0,0 +1 @@
+ff707716c0c4748ffeb21996aefa8d269b3eab5b
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/tika-parser-xmp-commons-2.9.2.jar.sha1 b/plugins/ingest-attachment/licenses/tika-parser-xmp-commons-2.9.2.jar.sha1
new file mode 100644
index 0000000000000..396e2655b14db
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/tika-parser-xmp-commons-2.9.2.jar.sha1
@@ -0,0 +1 @@
+69104107ff85194df5acf682178128771863e442
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/tika-parser-zip-commons-2.9.2.jar.sha1 b/plugins/ingest-attachment/licenses/tika-parser-zip-commons-2.9.2.jar.sha1
new file mode 100644
index 0000000000000..bda62033e4e8c
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/tika-parser-zip-commons-2.9.2.jar.sha1
@@ -0,0 +1 @@
+2fcea85a56f93a5c0cb81f3d6dd8673f3d81c598
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/tika-parsers-standard-package-2.6.0.jar.sha1 b/plugins/ingest-attachment/licenses/tika-parsers-standard-package-2.6.0.jar.sha1
deleted file mode 100644
index 83c0777fcbe8a..0000000000000
--- a/plugins/ingest-attachment/licenses/tika-parsers-standard-package-2.6.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-00980e70b1df13c1236b750f0ca1462edd5d7417
\ No newline at end of file
diff --git a/plugins/ingest-attachment/licenses/tika-parsers-standard-package-2.9.2.jar.sha1 b/plugins/ingest-attachment/licenses/tika-parsers-standard-package-2.9.2.jar.sha1
new file mode 100644
index 0000000000000..bb76974b6344e
--- /dev/null
+++ b/plugins/ingest-attachment/licenses/tika-parsers-standard-package-2.9.2.jar.sha1
@@ -0,0 +1 @@
+c8408deb51fa617ef4e912b4d161712e695d3a29
\ No newline at end of file
diff --git a/plugins/repository-azure/build.gradle b/plugins/repository-azure/build.gradle
index c7836170d658f..ff62c328c7e74 100644
--- a/plugins/repository-azure/build.gradle
+++ b/plugins/repository-azure/build.gradle
@@ -56,6 +56,21 @@ dependencies {
   api "io.netty:netty-transport-native-unix-common:${versions.netty}"
   implementation project(':modules:transport-netty4')
   api 'com.azure:azure-storage-blob:12.23.0'
+  api 'com.azure:azure-identity:1.11.4'
+  // Start of transitive dependencies for azure-identity
+  api 'com.microsoft.azure:msal4j-persistence-extension:1.2.0'
+  api "net.java.dev.jna:jna-platform:${versions.jna}"
+  api 'com.microsoft.azure:msal4j:1.14.3'
+  api 'com.nimbusds:oauth2-oidc-sdk:11.9.1'
+  api 'com.nimbusds:nimbus-jose-jwt:9.37.3'
+  api 'com.nimbusds:content-type:2.3'
+  api 'com.nimbusds:lang-tag:1.7'
+  // Both msal4j:1.14.3 and oauth2-oidc-sdk:11.9.1 has compile dependency on different versions of json-smart,
+  // selected the higher version which is 2.5.0
+  api 'net.minidev:json-smart:2.5.0'
+  api 'net.minidev:accessors-smart:2.5.0'
+  api "org.ow2.asm:asm:${versions.asm}"
+  // End of transitive dependencies for azure-identity
   api "io.projectreactor.netty:reactor-netty-core:${versions.reactor_netty}"
   api "io.projectreactor.netty:reactor-netty-http:${versions.reactor_netty}"
   api "org.slf4j:slf4j-api:${versions.slf4j}"
@@ -180,7 +195,76 @@ thirdPartyAudit {
     'io.micrometer.observation.ObservationHandler',
     'io.micrometer.observation.ObservationRegistry',
     'io.micrometer.observation.ObservationRegistry$ObservationConfig',
-    'io.micrometer.tracing.handler.DefaultTracingObservationHandler'
+    'io.micrometer.tracing.handler.DefaultTracingObservationHandler',
+    // Start of the list of classes from the optional compile/provided dependencies used in "com.nimbusds:oauth2-oidc-sdk".
+    'com.google.crypto.tink.subtle.Ed25519Sign',
+    'com.google.crypto.tink.subtle.Ed25519Sign$KeyPair',
+    'com.google.crypto.tink.subtle.Ed25519Verify',
+    'com.google.crypto.tink.subtle.X25519',
+    'com.google.crypto.tink.subtle.XChaCha20Poly1305',
+    'jakarta.servlet.ServletRequest',
+    'jakarta.servlet.http.HttpServletRequest',
+    'jakarta.servlet.http.HttpServletResponse',
+    'javax.servlet.ServletRequest',
+    'javax.servlet.http.HttpServletRequest',
+    'javax.servlet.http.HttpServletResponse',
+    // net.shibboleth.utilities:java-support.* is declared as optional in the plugin `bnd-maven-plugin` used in "com.nimbusds:oauth2-oidc-sdk"
+    // Worth nothing that, the latest dependency "net.shibboleth.utilities:java-support:8.0.0" has many vulnerabilities.
+    // Hence ignored.
+    'net.shibboleth.utilities.java.support.xml.SerializeSupport',
+    'org.bouncycastle.asn1.pkcs.PrivateKeyInfo',
+    'org.bouncycastle.asn1.x509.AlgorithmIdentifier',
+    'org.bouncycastle.asn1.x509.SubjectPublicKeyInfo',
+    'org.bouncycastle.cert.X509CertificateHolder',
+    'org.bouncycastle.cert.jcajce.JcaX509CertificateHolder',
+    'org.bouncycastle.cert.jcajce.JcaX509v3CertificateBuilder',
+    'org.bouncycastle.crypto.InvalidCipherTextException',
+    'org.bouncycastle.crypto.engines.AESEngine',
+    'org.bouncycastle.crypto.modes.GCMBlockCipher',
+    'org.bouncycastle.jcajce.provider.BouncyCastleFipsProvider',
+    'org.bouncycastle.jce.provider.BouncyCastleProvider',
+    'org.bouncycastle.openssl.PEMKeyPair',
+    'org.bouncycastle.openssl.PEMParser',
+    'org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter',
+    'org.bouncycastle.operator.jcajce.JcaContentSignerBuilder',
+    'org.cryptomator.siv.SivMode',
+    'org.opensaml.core.config.InitializationException',
+    'org.opensaml.core.config.InitializationService',
+    'org.opensaml.core.xml.XMLObject',
+    'org.opensaml.core.xml.XMLObjectBuilder',
+    'org.opensaml.core.xml.XMLObjectBuilderFactory',
+    'org.opensaml.core.xml.config.XMLObjectProviderRegistrySupport',
+    'org.opensaml.core.xml.io.Marshaller',
+    'org.opensaml.core.xml.io.MarshallerFactory',
+    'org.opensaml.core.xml.io.MarshallingException',
+    'org.opensaml.core.xml.io.Unmarshaller',
+    'org.opensaml.core.xml.io.UnmarshallerFactory',
+    'org.opensaml.core.xml.schema.XSString',
+    'org.opensaml.core.xml.schema.impl.XSStringBuilder',
+    'org.opensaml.saml.saml2.core.Assertion',
+    'org.opensaml.saml.saml2.core.Attribute',
+    'org.opensaml.saml.saml2.core.AttributeStatement',
+    'org.opensaml.saml.saml2.core.AttributeValue',
+    'org.opensaml.saml.saml2.core.Audience',
+    'org.opensaml.saml.saml2.core.AudienceRestriction',
+    'org.opensaml.saml.saml2.core.AuthnContext',
+    'org.opensaml.saml.saml2.core.AuthnContextClassRef',
+    'org.opensaml.saml.saml2.core.AuthnStatement',
+    'org.opensaml.saml.saml2.core.Conditions',
+    'org.opensaml.saml.saml2.core.Issuer',
+    'org.opensaml.saml.saml2.core.NameID',
+    'org.opensaml.saml.saml2.core.Subject',
+    'org.opensaml.saml.saml2.core.SubjectConfirmation',
+    'org.opensaml.saml.saml2.core.SubjectConfirmationData',
+    'org.opensaml.saml.security.impl.SAMLSignatureProfileValidator',
+    'org.opensaml.security.credential.BasicCredential',
+    'org.opensaml.security.credential.Credential',
+    'org.opensaml.security.credential.UsageType',
+    'org.opensaml.xmlsec.signature.Signature',
+    'org.opensaml.xmlsec.signature.support.SignatureException',
+    'org.opensaml.xmlsec.signature.support.SignatureValidator',
+    'org.opensaml.xmlsec.signature.support.Signer',
+    // End of the list of classes from the optional compile/provided dependencies used in "com.nimbusds:oauth2-oidc-sdk".
   )
 
   ignoreViolations(
diff --git a/plugins/repository-azure/licenses/accessors-smart-2.5.0.jar.sha1 b/plugins/repository-azure/licenses/accessors-smart-2.5.0.jar.sha1
new file mode 100644
index 0000000000000..1578c94fcdc7b
--- /dev/null
+++ b/plugins/repository-azure/licenses/accessors-smart-2.5.0.jar.sha1
@@ -0,0 +1 @@
+aca011492dfe9c26f4e0659028a4fe0970829dd8
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/accessors-smart-LICENSE.txt b/plugins/repository-azure/licenses/accessors-smart-LICENSE.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/plugins/repository-azure/licenses/accessors-smart-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/plugins/repository-azure/licenses/accessors-smart-NOTICE.txt b/plugins/repository-azure/licenses/accessors-smart-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/plugins/repository-azure/licenses/asm-9.7.jar.sha1 b/plugins/repository-azure/licenses/asm-9.7.jar.sha1
new file mode 100644
index 0000000000000..84c9a9703af6d
--- /dev/null
+++ b/plugins/repository-azure/licenses/asm-9.7.jar.sha1
@@ -0,0 +1 @@
+073d7b3086e14beb604ced229c302feff6449723
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/asm-LICENSE.txt b/plugins/repository-azure/licenses/asm-LICENSE.txt
new file mode 100644
index 0000000000000..c71bb7bac5d4d
--- /dev/null
+++ b/plugins/repository-azure/licenses/asm-LICENSE.txt
@@ -0,0 +1,27 @@
+ASM: a very small and fast Java bytecode manipulation framework
+Copyright (c) 2000-2011 INRIA, France Telecom
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+1. Redistributions of source code must retain the above copyright
+  notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright
+  notice, this list of conditions and the following disclaimer in the
+  documentation and/or other materials provided with the distribution.
+3. Neither the name of the copyright holders nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/plugins/repository-azure/licenses/asm-NOTICE.txt b/plugins/repository-azure/licenses/asm-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/plugins/repository-azure/licenses/azure-identity-1.11.4.jar.sha1 b/plugins/repository-azure/licenses/azure-identity-1.11.4.jar.sha1
new file mode 100644
index 0000000000000..c8d98ba9c8ad2
--- /dev/null
+++ b/plugins/repository-azure/licenses/azure-identity-1.11.4.jar.sha1
@@ -0,0 +1 @@
+59b5ce48888f638b80d85ef5aa0e22a265d3dc89
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/commons-lang3-3.13.0.jar.sha1 b/plugins/repository-azure/licenses/commons-lang3-3.13.0.jar.sha1
deleted file mode 100644
index d0c2f2486ee1f..0000000000000
--- a/plugins/repository-azure/licenses/commons-lang3-3.13.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b7263237aa89c1f99b327197c41d0669707a462e
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/commons-lang3-3.14.0.jar.sha1 b/plugins/repository-azure/licenses/commons-lang3-3.14.0.jar.sha1
new file mode 100644
index 0000000000000..d783e07e40902
--- /dev/null
+++ b/plugins/repository-azure/licenses/commons-lang3-3.14.0.jar.sha1
@@ -0,0 +1 @@
+1ed471194b02f2c6cb734a0cd6f6f107c673afae
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/content-type-2.3.jar.sha1 b/plugins/repository-azure/licenses/content-type-2.3.jar.sha1
new file mode 100644
index 0000000000000..e18bbaec9a89c
--- /dev/null
+++ b/plugins/repository-azure/licenses/content-type-2.3.jar.sha1
@@ -0,0 +1 @@
+e3aa0be212d7a42839a8f3f506f5b990bcce0222
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/content-type-LICENSE.txt b/plugins/repository-azure/licenses/content-type-LICENSE.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/plugins/repository-azure/licenses/content-type-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/plugins/repository-azure/licenses/content-type-NOTICE.txt b/plugins/repository-azure/licenses/content-type-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/plugins/repository-azure/licenses/jna-platform-5.13.0.jar.sha1 b/plugins/repository-azure/licenses/jna-platform-5.13.0.jar.sha1
new file mode 100644
index 0000000000000..e2a8ba1c1bbd3
--- /dev/null
+++ b/plugins/repository-azure/licenses/jna-platform-5.13.0.jar.sha1
@@ -0,0 +1 @@
+88e9a306715e9379f3122415ef4ae759a352640d
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/jna-platform-LICENSE.txt b/plugins/repository-azure/licenses/jna-platform-LICENSE.txt
new file mode 100644
index 0000000000000..c5a025f0c3e6d
--- /dev/null
+++ b/plugins/repository-azure/licenses/jna-platform-LICENSE.txt
@@ -0,0 +1,26 @@
+SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1
+
+Java Native Access (JNA) is licensed under the LGPL, version 2.1
+or later, or (from version 4.0 onward) the Apache License,
+version 2.0.
+
+You can freely decide which license you want to apply to the project.
+
+You may obtain a copy of the LGPL License at:
+
+http://www.gnu.org/licenses/licenses.html
+
+A copy is also included in the downloadable source code package
+containing JNA, in file "LGPL2.1", under the same directory
+as this file.
+
+You may obtain a copy of the Apache License at:
+
+http://www.apache.org/licenses/
+
+A copy is also included in the downloadable source code package
+containing JNA, in file "AL2.0", under the same directory
+as this file.
+
+Commercial support may be available, please e-mail
+twall[at]users[dot]sf[dot]net.
diff --git a/plugins/repository-azure/licenses/jna-platform-NOTICE.txt b/plugins/repository-azure/licenses/jna-platform-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/plugins/repository-hdfs/licenses/json-smart-2.5.0.jar.sha1 b/plugins/repository-azure/licenses/json-smart-2.5.0.jar.sha1
similarity index 100%
rename from plugins/repository-hdfs/licenses/json-smart-2.5.0.jar.sha1
rename to plugins/repository-azure/licenses/json-smart-2.5.0.jar.sha1
diff --git a/plugins/repository-azure/licenses/json-smart-LICENSE.txt b/plugins/repository-azure/licenses/json-smart-LICENSE.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/plugins/repository-azure/licenses/json-smart-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/plugins/repository-azure/licenses/json-smart-NOTICE.txt b/plugins/repository-azure/licenses/json-smart-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/plugins/repository-azure/licenses/lang-tag-1.7.jar.sha1 b/plugins/repository-azure/licenses/lang-tag-1.7.jar.sha1
new file mode 100644
index 0000000000000..9cd79d1dba715
--- /dev/null
+++ b/plugins/repository-azure/licenses/lang-tag-1.7.jar.sha1
@@ -0,0 +1 @@
+97c73ecd70bc7e8eefb26c5eea84f251a63f1031
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/lang-tag-LICENSE.txt b/plugins/repository-azure/licenses/lang-tag-LICENSE.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/plugins/repository-azure/licenses/lang-tag-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/plugins/repository-azure/licenses/lang-tag-NOTICE.txt b/plugins/repository-azure/licenses/lang-tag-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/plugins/repository-azure/licenses/msal4j-1.14.3.jar.sha1 b/plugins/repository-azure/licenses/msal4j-1.14.3.jar.sha1
new file mode 100644
index 0000000000000..2a6e42e3f2b48
--- /dev/null
+++ b/plugins/repository-azure/licenses/msal4j-1.14.3.jar.sha1
@@ -0,0 +1 @@
+117b28c41bd760f979ed1b6467c5ec491f0d4d60
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/msal4j-LICENSE.txt b/plugins/repository-azure/licenses/msal4j-LICENSE.txt
new file mode 100644
index 0000000000000..21071075c2459
--- /dev/null
+++ b/plugins/repository-azure/licenses/msal4j-LICENSE.txt
@@ -0,0 +1,21 @@
+    MIT License
+
+    Copyright (c) Microsoft Corporation. All rights reserved.
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE
diff --git a/plugins/repository-azure/licenses/msal4j-NOTICE.txt b/plugins/repository-azure/licenses/msal4j-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/plugins/repository-azure/licenses/msal4j-persistence-extension-1.2.0.jar.sha1 b/plugins/repository-azure/licenses/msal4j-persistence-extension-1.2.0.jar.sha1
new file mode 100644
index 0000000000000..cfcf7548b7694
--- /dev/null
+++ b/plugins/repository-azure/licenses/msal4j-persistence-extension-1.2.0.jar.sha1
@@ -0,0 +1 @@
+1111a95878de8745ddc9de132df18ebd9ca7024d
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/msal4j-persistence-extension-LICENSE.txt b/plugins/repository-azure/licenses/msal4j-persistence-extension-LICENSE.txt
new file mode 100644
index 0000000000000..21071075c2459
--- /dev/null
+++ b/plugins/repository-azure/licenses/msal4j-persistence-extension-LICENSE.txt
@@ -0,0 +1,21 @@
+    MIT License
+
+    Copyright (c) Microsoft Corporation. All rights reserved.
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is
+    furnished to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in all
+    copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+    OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+    SOFTWARE
diff --git a/plugins/repository-azure/licenses/msal4j-persistence-extension-NOTICE.txt b/plugins/repository-azure/licenses/msal4j-persistence-extension-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/plugins/repository-azure/licenses/netty-codec-dns-4.1.107.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-dns-4.1.107.Final.jar.sha1
deleted file mode 100644
index d823de7ffadd4..0000000000000
--- a/plugins/repository-azure/licenses/netty-codec-dns-4.1.107.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-945e8ad5ab7ec4f11fb0257d2594af0cfae1d4b7
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/netty-codec-dns-4.1.109.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-dns-4.1.109.Final.jar.sha1
new file mode 100644
index 0000000000000..b13a709f1c449
--- /dev/null
+++ b/plugins/repository-azure/licenses/netty-codec-dns-4.1.109.Final.jar.sha1
@@ -0,0 +1 @@
+ee231baee2cc9f1300ecc0d9a1e8bb9b31db02fa
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/netty-codec-http2-4.1.107.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-http2-4.1.107.Final.jar.sha1
deleted file mode 100644
index 5b3d3311edc9f..0000000000000
--- a/plugins/repository-azure/licenses/netty-codec-http2-4.1.107.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3885ffe7dd05c9773df70c61009f34a5a8a383ec
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/netty-codec-http2-4.1.109.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-http2-4.1.109.Final.jar.sha1
new file mode 100644
index 0000000000000..b83ad36222d07
--- /dev/null
+++ b/plugins/repository-azure/licenses/netty-codec-http2-4.1.109.Final.jar.sha1
@@ -0,0 +1 @@
+6bd4a54b69a81356393f6e4621bad40754f8a5a2
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/netty-codec-socks-4.1.107.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-socks-4.1.107.Final.jar.sha1
deleted file mode 100644
index 114d77a1bb95f..0000000000000
--- a/plugins/repository-azure/licenses/netty-codec-socks-4.1.107.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3b7070e9acfe262bb0bd936c4051116631796b3b
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/netty-codec-socks-4.1.109.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-codec-socks-4.1.109.Final.jar.sha1
new file mode 100644
index 0000000000000..5caf947d87a1b
--- /dev/null
+++ b/plugins/repository-azure/licenses/netty-codec-socks-4.1.109.Final.jar.sha1
@@ -0,0 +1 @@
+7f4f0c0dd54c578af2c613a0db7172bf7dca9c79
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/netty-handler-proxy-4.1.107.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-handler-proxy-4.1.107.Final.jar.sha1
deleted file mode 100644
index 5a4bde479eb38..0000000000000
--- a/plugins/repository-azure/licenses/netty-handler-proxy-4.1.107.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-ebc495e9b2bc2c9ab60a264b40f62dc0671d9f6e
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/netty-handler-proxy-4.1.109.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-handler-proxy-4.1.109.Final.jar.sha1
new file mode 100644
index 0000000000000..e0f52ab04ea84
--- /dev/null
+++ b/plugins/repository-azure/licenses/netty-handler-proxy-4.1.109.Final.jar.sha1
@@ -0,0 +1 @@
+a77224107f586a7f9e3dc5d12fc0d4d8f0c04803
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/netty-resolver-dns-4.1.107.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-resolver-dns-4.1.107.Final.jar.sha1
deleted file mode 100644
index a62cb0fefcc40..0000000000000
--- a/plugins/repository-azure/licenses/netty-resolver-dns-4.1.107.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d655d09e972dee46f580dbcf41c0d1356aea9e1b
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/netty-resolver-dns-4.1.109.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-resolver-dns-4.1.109.Final.jar.sha1
new file mode 100644
index 0000000000000..b42cdc2835eb0
--- /dev/null
+++ b/plugins/repository-azure/licenses/netty-resolver-dns-4.1.109.Final.jar.sha1
@@ -0,0 +1 @@
+5f4d858234b557b73631a24e562bb89fc5399cad
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.107.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.107.Final.jar.sha1
deleted file mode 100644
index 407ecaffdad30..0000000000000
--- a/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.107.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-4d61d4959741109b3eccd7337f11fc89fa90a74a
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.109.Final.jar.sha1 b/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.109.Final.jar.sha1
new file mode 100644
index 0000000000000..5afeb9627c9b5
--- /dev/null
+++ b/plugins/repository-azure/licenses/netty-transport-native-unix-common-4.1.109.Final.jar.sha1
@@ -0,0 +1 @@
+da7fe1e6943cbab8ee48df2beadc2c8304f347a2
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/nimbus-jose-jwt-9.37.3.jar.sha1 b/plugins/repository-azure/licenses/nimbus-jose-jwt-9.37.3.jar.sha1
new file mode 100644
index 0000000000000..7278cd8994f71
--- /dev/null
+++ b/plugins/repository-azure/licenses/nimbus-jose-jwt-9.37.3.jar.sha1
@@ -0,0 +1 @@
+700f71ffefd60c16bd8ce711a956967ea9071cec
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/nimbus-jose-jwt-LICENSE.txt b/plugins/repository-azure/licenses/nimbus-jose-jwt-LICENSE.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/plugins/repository-azure/licenses/nimbus-jose-jwt-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/plugins/repository-azure/licenses/nimbus-jose-jwt-NOTICE.txt b/plugins/repository-azure/licenses/nimbus-jose-jwt-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/plugins/repository-azure/licenses/oauth2-oidc-sdk-11.9.1.jar.sha1 b/plugins/repository-azure/licenses/oauth2-oidc-sdk-11.9.1.jar.sha1
new file mode 100644
index 0000000000000..96d9a196a172a
--- /dev/null
+++ b/plugins/repository-azure/licenses/oauth2-oidc-sdk-11.9.1.jar.sha1
@@ -0,0 +1 @@
+fa9a2e447e2cef4dfda40a854dd7ec35624a7799
\ No newline at end of file
diff --git a/plugins/repository-azure/licenses/oauth2-oidc-sdk-LICENSE.txt b/plugins/repository-azure/licenses/oauth2-oidc-sdk-LICENSE.txt
new file mode 100644
index 0000000000000..d645695673349
--- /dev/null
+++ b/plugins/repository-azure/licenses/oauth2-oidc-sdk-LICENSE.txt
@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/plugins/repository-azure/licenses/oauth2-oidc-sdk-NOTICE.txt b/plugins/repository-azure/licenses/oauth2-oidc-sdk-NOTICE.txt
new file mode 100644
index 0000000000000..e69de29bb2d1d
diff --git a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureBlobStore.java b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureBlobStore.java
index e76a6bdd16764..acaaa043df3ac 100644
--- a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureBlobStore.java
+++ b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureBlobStore.java
@@ -175,7 +175,7 @@ public BlobContainer blobContainer(BlobPath path) {
     }
 
     @Override
-    public void close() {
+    public void close() throws IOException {
         service.close();
     }
 
diff --git a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureRepositoryPlugin.java b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureRepositoryPlugin.java
index 78db7cb2d0ea7..aca213f9fed79 100644
--- a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureRepositoryPlugin.java
+++ b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureRepositoryPlugin.java
@@ -91,6 +91,7 @@ public List<Setting<?>> getSettings() {
             AzureStorageSettings.ACCOUNT_SETTING,
             AzureStorageSettings.KEY_SETTING,
             AzureStorageSettings.SAS_TOKEN_SETTING,
+            AzureStorageSettings.TOKEN_CREDENTIAL_TYPE_SETTING,
             AzureStorageSettings.ENDPOINT_SUFFIX_SETTING,
             AzureStorageSettings.TIMEOUT_SETTING,
             AzureStorageSettings.MAX_RETRIES_SETTING,
diff --git a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java
index 74edd4f3eb23c..f39ed185d8b35 100644
--- a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java
+++ b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageService.java
@@ -47,7 +47,6 @@
 import com.azure.storage.blob.BlobServiceClientBuilder;
 import com.azure.storage.blob.models.ParallelTransferOptions;
 import com.azure.storage.blob.specialized.BlockBlobAsyncClient;
-import com.azure.storage.common.implementation.connectionstring.StorageConnectionString;
 import com.azure.storage.common.implementation.connectionstring.StorageEndpoint;
 import com.azure.storage.common.policy.RequestRetryOptions;
 import com.azure.storage.common.policy.RetryPolicyType;
@@ -59,14 +58,19 @@
 import org.opensearch.core.common.unit.ByteSizeUnit;
 import org.opensearch.core.common.unit.ByteSizeValue;
 
+import java.io.IOException;
 import java.net.Authenticator;
 import java.net.PasswordAuthentication;
 import java.net.URISyntaxException;
+import java.security.AccessController;
 import java.security.InvalidKeyException;
+import java.security.PrivilegedAction;
 import java.time.Duration;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 import java.util.concurrent.ThreadFactory;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -99,6 +103,37 @@ public class AzureStorageService implements AutoCloseable {
     // 'package' for testing
     volatile Map<String, AzureStorageSettings> storageSettings = emptyMap();
     private final Map<AzureStorageSettings, ClientState> clients = new ConcurrentHashMap<>();
+    private final ExecutorService executor;
+
+    private static final class IdentityClientThreadFactory implements ThreadFactory {
+        final ThreadGroup group;
+        final AtomicInteger threadNumber = new AtomicInteger(1);
+        final String namePrefix;
+
+        @SuppressWarnings("removal")
+        IdentityClientThreadFactory(String namePrefix) {
+            this.namePrefix = namePrefix;
+            SecurityManager s = System.getSecurityManager();
+            group = (s != null) ? s.getThreadGroup() : Thread.currentThread().getThreadGroup();
+        }
+
+        @Override
+        public Thread newThread(Runnable r) {
+            Thread t = new Thread(group, new Runnable() {
+                @SuppressWarnings("removal")
+                public void run() {
+                    AccessController.doPrivileged(new PrivilegedAction<>() {
+                        public Void run() {
+                            r.run();
+                            return null;
+                        }
+                    });
+                }
+            }, namePrefix + "[T#" + threadNumber.getAndIncrement() + "]", 0);
+            t.setDaemon(true);
+            return t;
+        }
+    }
 
     static {
         // See please:
@@ -112,6 +147,9 @@ public AzureStorageService(Settings settings) {
         // eagerly load client settings so that secure settings are read
         final Map<String, AzureStorageSettings> clientsSettings = AzureStorageSettings.load(settings);
         refreshAndClearCache(clientsSettings);
+        executor = SocketAccess.doPrivilegedException(
+            () -> Executors.newCachedThreadPool(new IdentityClientThreadFactory("azure-identity-client"))
+        );
     }
 
     /**
@@ -162,7 +200,6 @@ public Tuple<BlobServiceClient, Supplier<Context>> client(String clientName, BiC
     private ClientState buildClient(AzureStorageSettings azureStorageSettings, BiConsumer<HttpRequest, HttpResponse> statsCollector)
         throws InvalidKeyException, URISyntaxException {
         final BlobServiceClientBuilder builder = createClientBuilder(azureStorageSettings);
-
         final NioEventLoopGroup eventLoopGroup = new NioEventLoopGroup(new NioThreadFactory());
         final NettyAsyncHttpClientBuilder clientBuilder = new NettyAsyncHttpClientBuilder().eventLoopGroup(eventLoopGroup);
 
@@ -216,8 +253,7 @@ protected PasswordAuthentication getPasswordAuthentication() {
      * <a href="https://github.com/Azure/azure-sdk-for-java/blob/main/sdk/storage/azure-storage-blob/migrationGuides/V8_V12.md#miscellaneous">migration guide</a> for mode details:
      */
     private BlobServiceClientBuilder applyLocationMode(final BlobServiceClientBuilder builder, final AzureStorageSettings settings) {
-        final StorageConnectionString storageConnectionString = StorageConnectionString.create(settings.getConnectString(), logger);
-        final StorageEndpoint endpoint = storageConnectionString.getBlobEndpoint();
+        final StorageEndpoint endpoint = settings.getStorageEndpoint(logger);
 
         if (endpoint == null || endpoint.getPrimaryUri() == null) {
             throw new IllegalArgumentException("connectionString missing required settings to derive blob service primary endpoint.");
@@ -247,9 +283,8 @@ private BlobServiceClientBuilder applyLocationMode(final BlobServiceClientBuilde
         return builder;
     }
 
-    private static BlobServiceClientBuilder createClientBuilder(AzureStorageSettings settings) throws InvalidKeyException,
-        URISyntaxException {
-        return SocketAccess.doPrivilegedException(() -> new BlobServiceClientBuilder().connectionString(settings.getConnectString()));
+    private BlobServiceClientBuilder createClientBuilder(AzureStorageSettings settings) throws InvalidKeyException, URISyntaxException {
+        return SocketAccess.doPrivilegedException(() -> settings.configure(new BlobServiceClientBuilder(), executor, logger));
     }
 
     /**
@@ -295,9 +330,19 @@ public Map<String, AzureStorageSettings> refreshAndClearCache(Map<String, AzureS
     }
 
     @Override
-    public void close() {
+    public void close() throws IOException {
         this.clients.values().forEach(this::closeInternally);
         this.clients.clear();
+        this.executor.shutdown();
+        try {
+            if (this.executor.awaitTermination(30, TimeUnit.SECONDS) == false) {
+                this.executor.shutdownNow();
+                logger.warning("The executor was not shutdown gracefully with 30 seconds");
+            }
+        } catch (final InterruptedException ex) {
+            Thread.currentThread().interrupt();
+            throw new IOException(ex);
+        }
     }
 
     public Duration getBlobRequestTimeout(String clientName) {
diff --git a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java
index e73ded679cf2b..1e09b89b8dadf 100644
--- a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java
+++ b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/AzureStorageSettings.java
@@ -32,7 +32,16 @@
 
 package org.opensearch.repositories.azure;
 
+import com.azure.core.util.logging.ClientLogger;
+import com.azure.identity.ManagedIdentityCredential;
+import com.azure.identity.ManagedIdentityCredentialBuilder;
+import com.azure.identity.implementation.CredentialBuilderBaseHelper;
+import com.azure.storage.blob.BlobServiceClientBuilder;
+import com.azure.storage.common.implementation.Constants;
+import com.azure.storage.common.implementation.connectionstring.StorageConnectionString;
+import com.azure.storage.common.implementation.connectionstring.StorageEndpoint;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.TriFunction;
 import org.opensearch.common.collect.MapBuilder;
 import org.opensearch.common.settings.SecureSetting;
 import org.opensearch.common.settings.Setting;
@@ -45,11 +54,14 @@
 import org.opensearch.core.common.settings.SecureString;
 
 import java.net.InetAddress;
+import java.net.URI;
 import java.net.UnknownHostException;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.Locale;
 import java.util.Map;
+import java.util.concurrent.ExecutorService;
+import java.util.function.Function;
 
 final class AzureStorageSettings {
 
@@ -77,6 +89,18 @@ final class AzureStorageSettings {
         key -> SecureSetting.secureString(key, null)
     );
 
+    /** Azure token credentials such as Managed Identity */
+    public static final AffixSetting<String> TOKEN_CREDENTIAL_TYPE_SETTING = Setting.affixKeySetting(
+        AZURE_CLIENT_PREFIX_KEY,
+        "token_credential_type",
+        key -> Setting.simpleString(key, value -> {
+            if (Strings.hasText(value) == true) {
+                TokenCredentialType.valueOfType(value);
+            }
+        }, Property.NodeScope),
+        () -> ACCOUNT_SETTING
+    );
+
     /** max_retries: Number of retries in case of Azure errors. Defaults to 3 (RetryPolicy.DEFAULT_CLIENT_RETRY_COUNT). */
     public static final AffixSetting<Integer> MAX_RETRIES_SETTING = Setting.affixKeySetting(
         AZURE_CLIENT_PREFIX_KEY,
@@ -194,7 +218,9 @@ final class AzureStorageSettings {
     );
 
     private final String account;
-    private final String connectString;
+    private final String tokenCredentialType;
+    private final TriFunction<BlobServiceClientBuilder, ExecutorService, ClientLogger, BlobServiceClientBuilder> clientBuilder;
+    private final Function<ClientLogger, StorageEndpoint> endpointBuilder;
     private final String endpointSuffix;
     private final TimeValue timeout;
     private final int maxRetries;
@@ -208,7 +234,9 @@ final class AzureStorageSettings {
     // copy-constructor
     private AzureStorageSettings(
         String account,
-        String connectString,
+        String tokenCredentialType,
+        TriFunction<BlobServiceClientBuilder, ExecutorService, ClientLogger, BlobServiceClientBuilder> clientBuilder,
+        Function<ClientLogger, StorageEndpoint> endpointBuilder,
         String endpointSuffix,
         TimeValue timeout,
         int maxRetries,
@@ -220,7 +248,9 @@ private AzureStorageSettings(
         ProxySettings proxySettings
     ) {
         this.account = account;
-        this.connectString = connectString;
+        this.tokenCredentialType = tokenCredentialType;
+        this.clientBuilder = clientBuilder;
+        this.endpointBuilder = endpointBuilder;
         this.endpointSuffix = endpointSuffix;
         this.timeout = timeout;
         this.maxRetries = maxRetries;
@@ -236,6 +266,7 @@ private AzureStorageSettings(
         String account,
         String key,
         String sasToken,
+        String tokenCredentialType,
         String endpointSuffix,
         TimeValue timeout,
         int maxRetries,
@@ -246,7 +277,37 @@ private AzureStorageSettings(
         ProxySettings proxySettings
     ) {
         this.account = account;
-        this.connectString = buildConnectString(account, key, sasToken, endpointSuffix);
+        this.tokenCredentialType = tokenCredentialType;
+        if (Strings.hasText(tokenCredentialType) == true) {
+            this.endpointBuilder = (logger) -> {
+                String tokenCredentialEndpointSuffix = endpointSuffix;
+                if (Strings.hasText(tokenCredentialEndpointSuffix) == false) {
+                    // Default to "core.windows.net".
+                    tokenCredentialEndpointSuffix = Constants.ConnectionStringConstants.DEFAULT_DNS;
+                }
+                final URI primaryBlobEndpoint = URI.create("https://" + account + ".blob." + tokenCredentialEndpointSuffix);
+                final URI secondaryBlobEndpoint = URI.create("https://" + account + "-secondary.blob." + tokenCredentialEndpointSuffix);
+                return new StorageEndpoint(primaryBlobEndpoint, secondaryBlobEndpoint);
+            };
+
+            this.clientBuilder = (builder, executor, logger) -> builder.credential(new ManagedIdentityCredentialBuilder() {
+                @Override
+                public ManagedIdentityCredential build() {
+                    // Use the privileged executor with IdentityClient instance
+                    CredentialBuilderBaseHelper.getClientOptions(this).setExecutorService(executor);
+                    return super.build();
+                }
+            }.build()).endpoint(endpointBuilder.apply(logger).getPrimaryUri());
+        } else {
+            final String connectString = buildConnectString(account, key, sasToken, endpointSuffix);
+
+            this.endpointBuilder = (logger) -> {
+                final StorageConnectionString storageConnectionString = StorageConnectionString.create(connectString, logger);
+                return storageConnectionString.getBlobEndpoint();
+            };
+
+            this.clientBuilder = (builder, executor, logger) -> builder.connectionString(connectString);
+        }
         this.endpointSuffix = endpointSuffix;
         this.timeout = timeout;
         this.maxRetries = maxRetries;
@@ -258,6 +319,14 @@ private AzureStorageSettings(
         this.proxySettings = proxySettings;
     }
 
+    public String getTokenCredentialType() {
+        return tokenCredentialType;
+    }
+
+    public StorageEndpoint getStorageEndpoint(ClientLogger logger) {
+        return endpointBuilder.apply(logger);
+    }
+
     public String getEndpointSuffix() {
         return endpointSuffix;
     }
@@ -274,10 +343,6 @@ public ProxySettings getProxySettings() {
         return proxySettings;
     }
 
-    public String getConnectString() {
-        return connectString;
-    }
-
     private static String buildConnectString(String account, @Nullable String key, @Nullable String sasToken, String endpointSuffix) {
         final boolean hasSasToken = Strings.hasText(sasToken);
         final boolean hasKey = Strings.hasText(key);
@@ -325,6 +390,7 @@ public String toString() {
         final StringBuilder sb = new StringBuilder("AzureStorageSettings{");
         sb.append("account='").append(account).append('\'');
         sb.append(", timeout=").append(timeout);
+        sb.append(", tokenCredentialType=").append(tokenCredentialType).append('\'');
         sb.append(", endpointSuffix='").append(endpointSuffix).append('\'');
         sb.append(", maxRetries=").append(maxRetries);
         sb.append(", proxySettings=").append(proxySettings != ProxySettings.NO_PROXY_SETTINGS ? "PROXY_SET" : "PROXY_NOT_SET");
@@ -370,6 +436,7 @@ private static AzureStorageSettings getClientSettings(Settings settings, String
                 account.toString(),
                 key.toString(),
                 sasToken.toString(),
+                getValue(settings, clientName, TOKEN_CREDENTIAL_TYPE_SETTING),
                 getValue(settings, clientName, ENDPOINT_SUFFIX_SETTING),
                 getValue(settings, clientName, TIMEOUT_SETTING),
                 getValue(settings, clientName, MAX_RETRIES_SETTING),
@@ -430,7 +497,9 @@ static Map<String, AzureStorageSettings> overrideLocationMode(
                 entry.getKey(),
                 new AzureStorageSettings(
                     entry.getValue().account,
-                    entry.getValue().connectString,
+                    entry.getValue().tokenCredentialType,
+                    entry.getValue().clientBuilder,
+                    entry.getValue().endpointBuilder,
                     entry.getValue().endpointSuffix,
                     entry.getValue().timeout,
                     entry.getValue().maxRetries,
@@ -445,4 +514,8 @@ static Map<String, AzureStorageSettings> overrideLocationMode(
         }
         return mapBuilder.immutableMap();
     }
+
+    public BlobServiceClientBuilder configure(BlobServiceClientBuilder builder, ExecutorService executor, ClientLogger logger) {
+        return clientBuilder.apply(builder, executor, logger);
+    }
 }
diff --git a/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/TokenCredentialType.java b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/TokenCredentialType.java
new file mode 100644
index 0000000000000..1f78f73934231
--- /dev/null
+++ b/plugins/repository-azure/src/main/java/org/opensearch/repositories/azure/TokenCredentialType.java
@@ -0,0 +1,40 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.repositories.azure;
+
+import java.util.Arrays;
+
+// Type of token credentials that the plugin supports
+public enum TokenCredentialType {
+    MANAGED_IDENTITY("managed");
+
+    private final String type;
+
+    TokenCredentialType(String type) {
+        this.type = type;
+    }
+
+    public static String[] getTokenCredentialTypes() {
+        return Arrays.stream(TokenCredentialType.values()).map(tokenCredentialType -> tokenCredentialType.type).toArray(String[]::new);
+    }
+
+    static TokenCredentialType valueOfType(String type) {
+        for (TokenCredentialType value : values()) {
+            if (value.type.equalsIgnoreCase(type) || value.name().equalsIgnoreCase(type)) {
+                return value;
+            }
+        }
+        throw new IllegalArgumentException(
+            "The token credential type '"
+                + type
+                + "' is unsupported, please use one of the following values: "
+                + String.join(", ", getTokenCredentialTypes())
+        );
+    }
+}
diff --git a/plugins/repository-azure/src/main/plugin-metadata/plugin-security.policy b/plugins/repository-azure/src/main/plugin-metadata/plugin-security.policy
index f3bf52ea46505..e8fbe35ebab1d 100644
--- a/plugins/repository-azure/src/main/plugin-metadata/plugin-security.policy
+++ b/plugins/repository-azure/src/main/plugin-metadata/plugin-security.policy
@@ -41,4 +41,7 @@ grant {
 
   // azure client set Authenticator for proxy username/password
   permission java.net.NetPermission "setDefaultAuthenticator";
+
+  // azure identity
+  permission java.util.PropertyPermission "os.name", "read";
 };
diff --git a/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java b/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java
index bb0eafc7d1d4a..ea74a49e593cf 100644
--- a/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java
+++ b/plugins/repository-azure/src/test/java/org/opensearch/repositories/azure/AzureStorageServiceTests.java
@@ -33,6 +33,7 @@
 package org.opensearch.repositories.azure;
 
 import com.azure.core.http.policy.HttpPipelinePolicy;
+import com.azure.identity.CredentialUnavailableException;
 import com.azure.storage.blob.BlobServiceClient;
 import com.azure.storage.common.policy.RequestRetryPolicy;
 import org.opensearch.common.settings.MockSecureSettings;
@@ -50,7 +51,6 @@
 import java.net.InetSocketAddress;
 import java.net.URI;
 import java.net.URISyntaxException;
-import java.net.UnknownHostException;
 import java.nio.charset.StandardCharsets;
 import java.time.Duration;
 import java.util.Base64;
@@ -106,11 +106,105 @@ public void testCreateClientWithEndpointSuffix() throws IOException {
             .put("azure.client.azure1.endpoint_suffix", "my_endpoint_suffix")
             .build();
         try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings)) {
-            final AzureStorageService azureStorageService = plugin.azureStoreService;
-            final BlobServiceClient client1 = azureStorageService.client("azure1").v1();
-            assertThat(client1.getAccountUrl(), equalTo("https://myaccount1.blob.my_endpoint_suffix"));
-            final BlobServiceClient client2 = azureStorageService.client("azure2").v1();
-            assertThat(client2.getAccountUrl(), equalTo("https://myaccount2.blob.core.windows.net"));
+            try (final AzureStorageService azureStorageService = plugin.azureStoreService) {
+                final BlobServiceClient client1 = azureStorageService.client("azure1").v1();
+                assertThat(client1.getAccountUrl(), equalTo("https://myaccount1.blob.my_endpoint_suffix"));
+                final BlobServiceClient client2 = azureStorageService.client("azure2").v1();
+                assertThat(client2.getAccountUrl(), equalTo("https://myaccount2.blob.core.windows.net"));
+            }
+        }
+    }
+
+    public void testCreateClientWithEndpointSuffixWhenManagedIdentityIsEnabled() throws IOException {
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        // Azure clients without account key and sas token.
+        secureSettings.setString("azure.client.azure1.account", "myaccount1");
+        secureSettings.setString("azure.client.azure2.account", "myaccount2");
+
+        final Settings settings = Settings.builder()
+            .setSecureSettings(secureSettings)
+            // Enabled managed identity for all clients
+            .put("azure.client.azure1.token_credential_type", TokenCredentialType.MANAGED_IDENTITY.name())
+            .put("azure.client.azure2.token_credential_type", TokenCredentialType.MANAGED_IDENTITY.name())
+            // Defined an endpoint suffix for azure client 1 only.
+            .put("azure.client.azure1.endpoint_suffix", "my_endpoint_suffix")
+            .build();
+        try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings)) {
+            try (final AzureStorageService azureStorageService = plugin.azureStoreService) {
+                // Expect azure client 1 to use the custom endpoint suffix
+                final BlobServiceClient client1 = azureStorageService.client("azure1").v1();
+                assertThat(client1.getAccountUrl(), equalTo("https://myaccount1.blob.my_endpoint_suffix"));
+                // Expect azure client 2 to use the default endpoint suffix
+                final BlobServiceClient client2 = azureStorageService.client("azure2").v1();
+                assertThat(client2.getAccountUrl(), equalTo("https://myaccount2.blob.core.windows.net"));
+            }
+        }
+    }
+
+    public void testCreateClientWithInvalidEndpointSuffix() throws IOException {
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("azure.client.azure1.account", "myaccount1");
+        secureSettings.setString("azure.client.azure2.account", "myaccount2");
+        secureSettings.setString("azure.client.azure2.key", encodeKey("mykey12"));
+        secureSettings.setString("azure.client.azure3.account", "myaccount1");
+        secureSettings.setString("azure.client.azure3.sas_token", encodeKey("mysastoken"));
+
+        final Settings settings = Settings.builder()
+            .setSecureSettings(secureSettings)
+            .put("azure.client.azure1.token_credential_type", TokenCredentialType.MANAGED_IDENTITY.name())
+            .put("azure.client.azure1.endpoint_suffix", "invalid endpoint suffix")
+            .put("azure.client.azure2.endpoint_suffix", "invalid endpoint suffix")
+            .put("azure.client.azure3.endpoint_suffix", "invalid endpoint suffix")
+            .build();
+
+        try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings)) {
+            try (final AzureStorageService azureStorageService = plugin.azureStoreService) {
+                // Expect all clients 1 to fail due to invalid endpoint suffix
+                expectThrows(SettingsException.class, () -> azureStorageService.client("azure1").v1());
+                expectThrows(RuntimeException.class, () -> azureStorageService.client("azure2").v1());
+                expectThrows(RuntimeException.class, () -> azureStorageService.client("azure3").v1());
+            }
+        }
+    }
+
+    public void testGettingSecondaryStorageBlobEndpoint() throws IOException {
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("azure.client.azure1.account", "myaccount1");
+        final Settings settings = Settings.builder()
+            .setSecureSettings(secureSettings)
+            // Enabled managed identity
+            .put("azure.client.azure1.token_credential_type", TokenCredentialType.MANAGED_IDENTITY.name())
+            .build();
+        try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings)) {
+            try (final AzureStorageService azureStorageService = plugin.azureStoreService) {
+                final Map<String, AzureStorageSettings> prevSettings = azureStorageService.refreshAndClearCache(Collections.emptyMap());
+                final Map<String, AzureStorageSettings> newSettings = AzureStorageSettings.overrideLocationMode(
+                    prevSettings,
+                    LocationMode.SECONDARY_ONLY
+                );
+                azureStorageService.refreshAndClearCache(newSettings);
+                final BlobServiceClient client1 = azureStorageService.client("azure1").v1();
+                assertThat(client1.getAccountUrl(), equalTo("https://myaccount1-secondary.blob.core.windows.net"));
+            }
+        }
+    }
+
+    public void testClientUsingManagedIdentity() throws IOException {
+        // Enabled managed identity
+        final Settings settings = Settings.builder()
+            .setSecureSettings(buildSecureSettings())
+            .put("azure.client.azure1.token_credential_type", TokenCredentialType.MANAGED_IDENTITY.name())
+            .build();
+        try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings)) {
+            try (final AzureStorageService azureStorageService = plugin.azureStoreService) {
+                final BlobServiceClient client1 = azureStorageService.client("azure1").v1();
+
+                // Expect the client to use managed identity for authentication, and it should fail because managed identity environment is
+                // not
+                // setup in the test
+                final CredentialUnavailableException e = expectThrows(CredentialUnavailableException.class, () -> client1.getAccountInfo());
+                assertThat(e.getMessage(), is("Managed Identity authentication is not available."));
+            }
         }
     }
 
@@ -128,29 +222,30 @@ public void testReinitClientSettings() throws IOException {
         secureSettings2.setString("azure.client.azure3.key", encodeKey("mykey23"));
         final Settings settings2 = Settings.builder().setSecureSettings(secureSettings2).build();
         try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings1)) {
-            final AzureStorageService azureStorageService = plugin.azureStoreService;
-            final BlobServiceClient client11 = azureStorageService.client("azure1").v1();
-            assertThat(client11.getAccountUrl(), equalTo("https://myaccount11.blob.core.windows.net"));
-            final BlobServiceClient client12 = azureStorageService.client("azure2").v1();
-            assertThat(client12.getAccountUrl(), equalTo("https://myaccount12.blob.core.windows.net"));
-            // client 3 is missing
-            final SettingsException e1 = expectThrows(SettingsException.class, () -> azureStorageService.client("azure3"));
-            assertThat(e1.getMessage(), is("Unable to find client with name [azure3]"));
-            // update client settings
-            plugin.reload(settings2);
-            // old client 1 not changed
-            assertThat(client11.getAccountUrl(), equalTo("https://myaccount11.blob.core.windows.net"));
-            // new client 1 is changed
-            final BlobServiceClient client21 = azureStorageService.client("azure1").v1();
-            assertThat(client21.getAccountUrl(), equalTo("https://myaccount21.blob.core.windows.net"));
-            // old client 2 not changed
-            assertThat(client12.getAccountUrl(), equalTo("https://myaccount12.blob.core.windows.net"));
-            // new client2 is gone
-            final SettingsException e2 = expectThrows(SettingsException.class, () -> azureStorageService.client("azure2"));
-            assertThat(e2.getMessage(), is("Unable to find client with name [azure2]"));
-            // client 3 emerged
-            final BlobServiceClient client23 = azureStorageService.client("azure3").v1();
-            assertThat(client23.getAccountUrl(), equalTo("https://myaccount23.blob.core.windows.net"));
+            try (final AzureStorageService azureStorageService = plugin.azureStoreService) {
+                final BlobServiceClient client11 = azureStorageService.client("azure1").v1();
+                assertThat(client11.getAccountUrl(), equalTo("https://myaccount11.blob.core.windows.net"));
+                final BlobServiceClient client12 = azureStorageService.client("azure2").v1();
+                assertThat(client12.getAccountUrl(), equalTo("https://myaccount12.blob.core.windows.net"));
+                // client 3 is missing
+                final SettingsException e1 = expectThrows(SettingsException.class, () -> azureStorageService.client("azure3"));
+                assertThat(e1.getMessage(), is("Unable to find client with name [azure3]"));
+                // update client settings
+                plugin.reload(settings2);
+                // old client 1 not changed
+                assertThat(client11.getAccountUrl(), equalTo("https://myaccount11.blob.core.windows.net"));
+                // new client 1 is changed
+                final BlobServiceClient client21 = azureStorageService.client("azure1").v1();
+                assertThat(client21.getAccountUrl(), equalTo("https://myaccount21.blob.core.windows.net"));
+                // old client 2 not changed
+                assertThat(client12.getAccountUrl(), equalTo("https://myaccount12.blob.core.windows.net"));
+                // new client2 is gone
+                final SettingsException e2 = expectThrows(SettingsException.class, () -> azureStorageService.client("azure2"));
+                assertThat(e2.getMessage(), is("Unable to find client with name [azure2]"));
+                // client 3 emerged
+                final BlobServiceClient client23 = azureStorageService.client("azure3").v1();
+                assertThat(client23.getAccountUrl(), equalTo("https://myaccount23.blob.core.windows.net"));
+            }
         }
     }
 
@@ -160,17 +255,18 @@ public void testReinitClientEmptySettings() throws IOException {
         secureSettings.setString("azure.client.azure1.key", encodeKey("mykey11"));
         final Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
         try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings)) {
-            final AzureStorageService azureStorageService = plugin.azureStoreService;
-            final BlobServiceClient client11 = azureStorageService.client("azure1").v1();
-            assertThat(client11.getAccountUrl(), equalTo("https://myaccount1.blob.core.windows.net"));
-            // reinit with empty settings
-            final SettingsException e = expectThrows(SettingsException.class, () -> plugin.reload(Settings.EMPTY));
-            assertThat(e.getMessage(), is("If you want to use an azure repository, you need to define a client configuration."));
-            // existing client untouched
-            assertThat(client11.getAccountUrl(), equalTo("https://myaccount1.blob.core.windows.net"));
-            // new client also untouched
-            final BlobServiceClient client21 = azureStorageService.client("azure1").v1();
-            assertThat(client21.getAccountUrl(), equalTo("https://myaccount1.blob.core.windows.net"));
+            try (final AzureStorageService azureStorageService = plugin.azureStoreService) {
+                final BlobServiceClient client11 = azureStorageService.client("azure1").v1();
+                assertThat(client11.getAccountUrl(), equalTo("https://myaccount1.blob.core.windows.net"));
+                // reinit with empty settings
+                final SettingsException e = expectThrows(SettingsException.class, () -> plugin.reload(Settings.EMPTY));
+                assertThat(e.getMessage(), is("If you want to use an azure repository, you need to define a client configuration."));
+                // existing client untouched
+                assertThat(client11.getAccountUrl(), equalTo("https://myaccount1.blob.core.windows.net"));
+                // new client also untouched
+                final BlobServiceClient client21 = azureStorageService.client("azure1").v1();
+                assertThat(client21.getAccountUrl(), equalTo("https://myaccount1.blob.core.windows.net"));
+            }
         }
     }
 
@@ -189,138 +285,150 @@ public void testReinitClientWrongSettings() throws IOException {
         secureSettings3.setString("azure.client.azure1.sas_token", encodeKey("mysasToken33"));
         final Settings settings3 = Settings.builder().setSecureSettings(secureSettings3).build();
         try (AzureRepositoryPlugin plugin = pluginWithSettingsValidation(settings1)) {
-            final AzureStorageService azureStorageService = plugin.azureStoreService;
-            final BlobServiceClient client11 = azureStorageService.client("azure1").v1();
-            assertThat(client11.getAccountUrl(), equalTo("https://myaccount1.blob.core.windows.net"));
-            final SettingsException e1 = expectThrows(SettingsException.class, () -> plugin.reload(settings2));
-            assertThat(e1.getMessage(), is("Neither a secret key nor a shared access token was set."));
-            final SettingsException e2 = expectThrows(SettingsException.class, () -> plugin.reload(settings3));
-            assertThat(e2.getMessage(), is("Both a secret as well as a shared access token were set."));
-            // existing client untouched
-            assertThat(client11.getAccountUrl(), equalTo("https://myaccount1.blob.core.windows.net"));
+            try (final AzureStorageService azureStorageService = plugin.azureStoreService) {
+                final BlobServiceClient client11 = azureStorageService.client("azure1").v1();
+                assertThat(client11.getAccountUrl(), equalTo("https://myaccount1.blob.core.windows.net"));
+                final SettingsException e1 = expectThrows(SettingsException.class, () -> plugin.reload(settings2));
+                assertThat(e1.getMessage(), is("Neither a secret key nor a shared access token was set."));
+                final SettingsException e2 = expectThrows(SettingsException.class, () -> plugin.reload(settings3));
+                assertThat(e2.getMessage(), is("Both a secret as well as a shared access token were set."));
+                // existing client untouched
+                assertThat(client11.getAccountUrl(), equalTo("https://myaccount1.blob.core.windows.net"));
+            }
         }
     }
 
-    public void testGetSelectedClientNonExisting() {
-        final AzureStorageService azureStorageService = storageServiceWithSettingsValidation(buildSettings());
-        final SettingsException e = expectThrows(SettingsException.class, () -> azureStorageService.client("azure4"));
-        assertThat(e.getMessage(), is("Unable to find client with name [azure4]"));
+    public void testGetSelectedClientNonExisting() throws IOException {
+        try (final AzureStorageService azureStorageService = storageServiceWithSettingsValidation(buildSettings())) {
+            final SettingsException e = expectThrows(SettingsException.class, () -> azureStorageService.client("azure4"));
+            assertThat(e.getMessage(), is("Unable to find client with name [azure4]"));
+        }
     }
 
-    public void testGetSelectedClientDefaultTimeout() {
+    public void testGetSelectedClientDefaultTimeout() throws IOException {
         final Settings timeoutSettings = Settings.builder()
             .setSecureSettings(buildSecureSettings())
             .put("azure.client.azure3.timeout", "30s")
             .build();
-        final AzureStorageService azureStorageService = storageServiceWithSettingsValidation(timeoutSettings);
-        assertThat(azureStorageService.getBlobRequestTimeout("azure1"), nullValue());
-        assertThat(azureStorageService.getBlobRequestTimeout("azure3"), is(Duration.ofSeconds(30)));
+        try (final AzureStorageService azureStorageService = storageServiceWithSettingsValidation(timeoutSettings)) {
+            assertThat(azureStorageService.getBlobRequestTimeout("azure1"), nullValue());
+            assertThat(azureStorageService.getBlobRequestTimeout("azure3"), is(Duration.ofSeconds(30)));
+        }
     }
 
-    public void testClientDefaultConnectTimeout() {
+    public void testClientDefaultConnectTimeout() throws IOException {
         final Settings settings = Settings.builder()
             .setSecureSettings(buildSecureSettings())
             .put("azure.client.azure3.connect.timeout", "25s")
             .build();
-        final AzureStorageService mock = storageServiceWithSettingsValidation(settings);
-        final TimeValue timeout = mock.storageSettings.get("azure3").getConnectTimeout();
+        try (final AzureStorageService mock = storageServiceWithSettingsValidation(settings)) {
+            final TimeValue timeout = mock.storageSettings.get("azure3").getConnectTimeout();
 
-        assertThat(timeout, notNullValue());
-        assertThat(timeout, equalTo(TimeValue.timeValueSeconds(25)));
-        assertThat(mock.storageSettings.get("azure2").getConnectTimeout(), notNullValue());
-        assertThat(mock.storageSettings.get("azure2").getConnectTimeout(), equalTo(TimeValue.timeValueSeconds(10)));
+            assertThat(timeout, notNullValue());
+            assertThat(timeout, equalTo(TimeValue.timeValueSeconds(25)));
+            assertThat(mock.storageSettings.get("azure2").getConnectTimeout(), notNullValue());
+            assertThat(mock.storageSettings.get("azure2").getConnectTimeout(), equalTo(TimeValue.timeValueSeconds(10)));
+        }
     }
 
-    public void testClientDefaultWriteTimeout() {
+    public void testClientDefaultWriteTimeout() throws IOException {
         final Settings settings = Settings.builder()
             .setSecureSettings(buildSecureSettings())
             .put("azure.client.azure3.write.timeout", "85s")
             .build();
-        final AzureStorageService mock = storageServiceWithSettingsValidation(settings);
-        final TimeValue timeout = mock.storageSettings.get("azure3").getWriteTimeout();
+        try (final AzureStorageService mock = storageServiceWithSettingsValidation(settings)) {
+            final TimeValue timeout = mock.storageSettings.get("azure3").getWriteTimeout();
 
-        assertThat(timeout, notNullValue());
-        assertThat(timeout, equalTo(TimeValue.timeValueSeconds(85)));
-        assertThat(mock.storageSettings.get("azure2").getWriteTimeout(), notNullValue());
-        assertThat(mock.storageSettings.get("azure2").getWriteTimeout(), equalTo(TimeValue.timeValueSeconds(60)));
+            assertThat(timeout, notNullValue());
+            assertThat(timeout, equalTo(TimeValue.timeValueSeconds(85)));
+            assertThat(mock.storageSettings.get("azure2").getWriteTimeout(), notNullValue());
+            assertThat(mock.storageSettings.get("azure2").getWriteTimeout(), equalTo(TimeValue.timeValueSeconds(60)));
+        }
     }
 
-    public void testClientDefaultReadTimeout() {
+    public void testClientDefaultReadTimeout() throws IOException {
         final Settings settings = Settings.builder()
             .setSecureSettings(buildSecureSettings())
             .put("azure.client.azure3.read.timeout", "120s")
             .build();
-        final AzureStorageService mock = storageServiceWithSettingsValidation(settings);
-        final TimeValue timeout = mock.storageSettings.get("azure3").getReadTimeout();
+        try (final AzureStorageService mock = storageServiceWithSettingsValidation(settings)) {
+            final TimeValue timeout = mock.storageSettings.get("azure3").getReadTimeout();
 
-        assertThat(timeout, notNullValue());
-        assertThat(timeout, equalTo(TimeValue.timeValueSeconds(120)));
-        assertThat(mock.storageSettings.get("azure2").getReadTimeout(), notNullValue());
-        assertThat(mock.storageSettings.get("azure2").getReadTimeout(), equalTo(TimeValue.timeValueSeconds(60)));
+            assertThat(timeout, notNullValue());
+            assertThat(timeout, equalTo(TimeValue.timeValueSeconds(120)));
+            assertThat(mock.storageSettings.get("azure2").getReadTimeout(), notNullValue());
+            assertThat(mock.storageSettings.get("azure2").getReadTimeout(), equalTo(TimeValue.timeValueSeconds(60)));
+        }
     }
 
-    public void testClientDefaultResponseTimeout() {
+    public void testClientDefaultResponseTimeout() throws IOException {
         final Settings settings = Settings.builder()
             .setSecureSettings(buildSecureSettings())
             .put("azure.client.azure3.response.timeout", "1ms")
             .build();
-        final AzureStorageService mock = storageServiceWithSettingsValidation(settings);
-        final TimeValue timeout = mock.storageSettings.get("azure3").getResponseTimeout();
+        try (final AzureStorageService mock = storageServiceWithSettingsValidation(settings)) {
+            final TimeValue timeout = mock.storageSettings.get("azure3").getResponseTimeout();
 
-        assertThat(timeout, notNullValue());
-        assertThat(timeout, equalTo(TimeValue.timeValueMillis(1)));
-        assertThat(mock.storageSettings.get("azure2").getResponseTimeout(), notNullValue());
-        assertThat(mock.storageSettings.get("azure2").getResponseTimeout(), equalTo(TimeValue.timeValueSeconds(60)));
+            assertThat(timeout, notNullValue());
+            assertThat(timeout, equalTo(TimeValue.timeValueMillis(1)));
+            assertThat(mock.storageSettings.get("azure2").getResponseTimeout(), notNullValue());
+            assertThat(mock.storageSettings.get("azure2").getResponseTimeout(), equalTo(TimeValue.timeValueSeconds(60)));
+        }
     }
 
-    public void testGetSelectedClientNoTimeout() {
-        final AzureStorageService azureStorageService = storageServiceWithSettingsValidation(buildSettings());
-        assertThat(azureStorageService.getBlobRequestTimeout("azure1"), nullValue());
+    public void testGetSelectedClientNoTimeout() throws IOException {
+        try (final AzureStorageService azureStorageService = storageServiceWithSettingsValidation(buildSettings())) {
+            assertThat(azureStorageService.getBlobRequestTimeout("azure1"), nullValue());
+        }
     }
 
-    public void testGetSelectedClientBackoffPolicy() {
-        final AzureStorageService azureStorageService = storageServiceWithSettingsValidation(buildSettings());
-        final BlobServiceClient client1 = azureStorageService.client("azure1").v1();
-        assertThat(requestRetryOptions(client1), is(notNullValue()));
+    public void testGetSelectedClientBackoffPolicy() throws IOException {
+        try (final AzureStorageService azureStorageService = storageServiceWithSettingsValidation(buildSettings())) {
+            final BlobServiceClient client1 = azureStorageService.client("azure1").v1();
+            assertThat(requestRetryOptions(client1), is(notNullValue()));
+        }
     }
 
-    public void testGetSelectedClientBackoffPolicyNbRetries() {
+    public void testGetSelectedClientBackoffPolicyNbRetries() throws IOException {
         final Settings timeoutSettings = Settings.builder()
             .setSecureSettings(buildSecureSettings())
             .put("azure.client.azure1.max_retries", 7)
             .build();
 
-        final AzureStorageService azureStorageService = storageServiceWithSettingsValidation(timeoutSettings);
-        final BlobServiceClient client1 = azureStorageService.client("azure1").v1();
-        assertThat(requestRetryOptions(client1), is(notNullValue()));
+        try (final AzureStorageService azureStorageService = storageServiceWithSettingsValidation(timeoutSettings)) {
+            final BlobServiceClient client1 = azureStorageService.client("azure1").v1();
+            assertThat(requestRetryOptions(client1), is(notNullValue()));
+        }
     }
 
-    public void testNoProxy() {
+    public void testNoProxy() throws IOException {
         final Settings settings = Settings.builder().setSecureSettings(buildSecureSettings()).build();
-        final AzureStorageService mock = storageServiceWithSettingsValidation(settings);
-        assertEquals(mock.storageSettings.get("azure1").getProxySettings(), ProxySettings.NO_PROXY_SETTINGS);
-        assertEquals(mock.storageSettings.get("azure2").getProxySettings(), ProxySettings.NO_PROXY_SETTINGS);
-        assertEquals(mock.storageSettings.get("azure3").getProxySettings(), ProxySettings.NO_PROXY_SETTINGS);
+        try (final AzureStorageService mock = storageServiceWithSettingsValidation(settings)) {
+            assertEquals(mock.storageSettings.get("azure1").getProxySettings(), ProxySettings.NO_PROXY_SETTINGS);
+            assertEquals(mock.storageSettings.get("azure2").getProxySettings(), ProxySettings.NO_PROXY_SETTINGS);
+            assertEquals(mock.storageSettings.get("azure3").getProxySettings(), ProxySettings.NO_PROXY_SETTINGS);
+        }
     }
 
-    public void testProxyHttp() throws UnknownHostException {
+    public void testProxyHttp() throws IOException {
         final Settings settings = Settings.builder()
             .setSecureSettings(buildSecureSettings())
             .put("azure.client.azure1.proxy.host", "127.0.0.1")
             .put("azure.client.azure1.proxy.port", 8080)
             .put("azure.client.azure1.proxy.type", "http")
             .build();
-        final AzureStorageService mock = storageServiceWithSettingsValidation(settings);
-        final ProxySettings azure1Proxy = mock.storageSettings.get("azure1").getProxySettings();
-
-        assertThat(azure1Proxy, notNullValue());
-        assertThat(azure1Proxy.getType(), is(ProxySettings.ProxyType.HTTP));
-        assertThat(azure1Proxy.getAddress(), is(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 8080)));
-        assertEquals(ProxySettings.NO_PROXY_SETTINGS, mock.storageSettings.get("azure2").getProxySettings());
-        assertEquals(ProxySettings.NO_PROXY_SETTINGS, mock.storageSettings.get("azure3").getProxySettings());
+        try (final AzureStorageService mock = storageServiceWithSettingsValidation(settings)) {
+            final ProxySettings azure1Proxy = mock.storageSettings.get("azure1").getProxySettings();
+
+            assertThat(azure1Proxy, notNullValue());
+            assertThat(azure1Proxy.getType(), is(ProxySettings.ProxyType.HTTP));
+            assertThat(azure1Proxy.getAddress(), is(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 8080)));
+            assertEquals(ProxySettings.NO_PROXY_SETTINGS, mock.storageSettings.get("azure2").getProxySettings());
+            assertEquals(ProxySettings.NO_PROXY_SETTINGS, mock.storageSettings.get("azure3").getProxySettings());
+        }
     }
 
-    public void testMultipleProxies() throws UnknownHostException {
+    public void testMultipleProxies() throws IOException {
         final Settings settings = Settings.builder()
             .setSecureSettings(buildSecureSettings())
             .put("azure.client.azure1.proxy.host", "127.0.0.1")
@@ -330,21 +438,22 @@ public void testMultipleProxies() throws UnknownHostException {
             .put("azure.client.azure2.proxy.port", 8081)
             .put("azure.client.azure2.proxy.type", "http")
             .build();
-        final AzureStorageService mock = storageServiceWithSettingsValidation(settings);
-        final ProxySettings azure1Proxy = mock.storageSettings.get("azure1").getProxySettings();
-        assertThat(azure1Proxy, notNullValue());
-        assertThat(azure1Proxy.getType(), is(ProxySettings.ProxyType.HTTP));
-        assertThat(azure1Proxy.getAddress(), is(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 8080)));
-        final ProxySettings azure2Proxy = mock.storageSettings.get("azure2").getProxySettings();
-        assertThat(azure2Proxy, notNullValue());
-        assertThat(azure2Proxy.getType(), is(ProxySettings.ProxyType.HTTP));
-        assertThat(azure2Proxy.getAddress(), is(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 8081)));
-        assertTrue(Strings.isNullOrEmpty(azure2Proxy.getUsername()));
-        assertTrue(Strings.isNullOrEmpty(azure2Proxy.getPassword()));
-        assertEquals(mock.storageSettings.get("azure3").getProxySettings(), ProxySettings.NO_PROXY_SETTINGS);
-    }
-
-    public void testProxySocks() throws UnknownHostException {
+        try (final AzureStorageService mock = storageServiceWithSettingsValidation(settings)) {
+            final ProxySettings azure1Proxy = mock.storageSettings.get("azure1").getProxySettings();
+            assertThat(azure1Proxy, notNullValue());
+            assertThat(azure1Proxy.getType(), is(ProxySettings.ProxyType.HTTP));
+            assertThat(azure1Proxy.getAddress(), is(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 8080)));
+            final ProxySettings azure2Proxy = mock.storageSettings.get("azure2").getProxySettings();
+            assertThat(azure2Proxy, notNullValue());
+            assertThat(azure2Proxy.getType(), is(ProxySettings.ProxyType.HTTP));
+            assertThat(azure2Proxy.getAddress(), is(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 8081)));
+            assertTrue(Strings.isNullOrEmpty(azure2Proxy.getUsername()));
+            assertTrue(Strings.isNullOrEmpty(azure2Proxy.getPassword()));
+            assertEquals(mock.storageSettings.get("azure3").getProxySettings(), ProxySettings.NO_PROXY_SETTINGS);
+        }
+    }
+
+    public void testProxySocks() throws IOException {
         final MockSecureSettings secureSettings = buildSecureSettings();
         secureSettings.setString("azure.client.azure1.proxy.username", "user");
         secureSettings.setString("azure.client.azure1.proxy.password", "pwd");
@@ -354,15 +463,16 @@ public void testProxySocks() throws UnknownHostException {
             .put("azure.client.azure1.proxy.type", "socks5")
             .setSecureSettings(secureSettings)
             .build();
-        final AzureStorageService mock = storageServiceWithSettingsValidation(settings);
-        final ProxySettings azure1Proxy = mock.storageSettings.get("azure1").getProxySettings();
-        assertThat(azure1Proxy, notNullValue());
-        assertThat(azure1Proxy.getType(), is(ProxySettings.ProxyType.SOCKS5));
-        assertThat(azure1Proxy.getAddress(), is(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 8080)));
-        assertEquals("user", azure1Proxy.getUsername());
-        assertEquals("pwd", azure1Proxy.getPassword());
-        assertEquals(ProxySettings.NO_PROXY_SETTINGS, mock.storageSettings.get("azure2").getProxySettings());
-        assertEquals(ProxySettings.NO_PROXY_SETTINGS, mock.storageSettings.get("azure3").getProxySettings());
+        try (final AzureStorageService mock = storageServiceWithSettingsValidation(settings)) {
+            final ProxySettings azure1Proxy = mock.storageSettings.get("azure1").getProxySettings();
+            assertThat(azure1Proxy, notNullValue());
+            assertThat(azure1Proxy.getType(), is(ProxySettings.ProxyType.SOCKS5));
+            assertThat(azure1Proxy.getAddress(), is(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), 8080)));
+            assertEquals("user", azure1Proxy.getUsername());
+            assertEquals("pwd", azure1Proxy.getPassword());
+            assertEquals(ProxySettings.NO_PROXY_SETTINGS, mock.storageSettings.get("azure2").getProxySettings());
+            assertEquals(ProxySettings.NO_PROXY_SETTINGS, mock.storageSettings.get("azure3").getProxySettings());
+        }
     }
 
     public void testProxyNoHost() {
@@ -420,6 +530,199 @@ public void testBlobNameFromUri() throws URISyntaxException {
         assertThat(name, is("path/to/myfile"));
     }
 
+    public void testSettingTokenCredentialForAuthenticationIsCaseInsensitive() throws IOException {
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        // Azure client without account key or sas token.
+        secureSettings.setString("azure.client.azure.account", "myaccount");
+        secureSettings.setString("azure.client.azure2.account", "myaccount");
+
+        // Enabled Managed Identity in the settings using lower case and mixed case
+        final Settings settings = Settings.builder()
+            .setSecureSettings(secureSettings)
+            .put("azure.client.azure.token_credential_type", "managed_identity")
+            .put("azure.client.azure2.token_credential_type", "managed_IDENTITY")
+            .build();
+
+        try (final AzureStorageService mock = storageServiceWithSettingsValidation(settings)) {
+            assertEquals(mock.storageSettings.get("azure").getTokenCredentialType(), "managed_identity");
+            assertEquals(mock.storageSettings.get("azure2").getTokenCredentialType(), "managed_IDENTITY");
+        }
+    }
+
+    public void testSettingTokenCredentialForAuthenticationWithAlternativeEnumValue() throws IOException {
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        // Azure client without account key or sas token.
+        secureSettings.setString("azure.client.azure.account", "myaccount");
+
+        // Enabled Managed Identity in the settings using lower case
+        final Settings settings = Settings.builder()
+            .setSecureSettings(secureSettings)
+            .put("azure.client.azure.token_credential_type", "managed")
+            .build();
+
+        try (final AzureStorageService mock = storageServiceWithSettingsValidation(settings)) {
+            assertEquals(mock.storageSettings.get("azure").getTokenCredentialType(), "managed");
+        }
+    }
+
+    public void testSettingUnsupportedTokenCredentialForAuthentication() {
+        final String unsupported_token_credential_type = "TOKEN_CREDENTIAL_TYPE_THAT_DOES_NOT_EXIST";
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+
+        // Azure client without account key or sas token.
+        secureSettings.setString("azure.client.azure.account", "myaccount");
+
+        // Enable the unsupported token credential type
+        final Settings settings = Settings.builder()
+            .setSecureSettings(secureSettings)
+            .put("azure.client.azure.token_credential_type", unsupported_token_credential_type)
+            .build();
+
+        final IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
+            () -> storageServiceWithSettingsValidation(settings)
+        );
+        assertEquals(
+            "The token credential type '"
+                + unsupported_token_credential_type
+                + "' is unsupported, please use one of the following values: "
+                + String.join(", ", TokenCredentialType.getTokenCredentialTypes()),
+            e.getMessage()
+        );
+    }
+
+    public void testTokenCredentialAuthenticationOverridesOtherFormOfAuthentications() throws IOException {
+        final String token_credential_type = TokenCredentialType.MANAGED_IDENTITY.name();
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        // Azure1 with account key
+        secureSettings.setString("azure.client.azure1.account", "myaccount1");
+        secureSettings.setString("azure.client.azure1.key", encodeKey("mykey"));
+
+        // Azure 2 with sas token
+        secureSettings.setString("azure.client.azure2.account", "myaccount2");
+        secureSettings.setString("azure.client.azure2.sas_token", encodeKey("mysastoken"));
+
+        // Azure 3 with account key and sas token
+        secureSettings.setString("azure.client.azure3.account", "myaccount3");
+        secureSettings.setString("azure.client.azure3.key", encodeKey("mykey"));
+        secureSettings.setString("azure.client.azure3.sas_token", encodeKey("mysastoken"));
+
+        // Azure 4 without sas token and account key
+        secureSettings.setString("azure.client.azure4.account", "myaccount4");
+
+        // Enable Managed Identity in all azure clients
+        final Settings settings = Settings.builder()
+            .setSecureSettings(secureSettings)
+            .put("azure.client.azure1.token_credential_type", token_credential_type)
+            .put("azure.client.azure2.token_credential_type", token_credential_type)
+            .put("azure.client.azure3.token_credential_type", token_credential_type)
+            .put("azure.client.azure4.token_credential_type", token_credential_type)
+            .build();
+        try (final AzureStorageService mock = storageServiceWithSettingsValidation(settings)) {
+            // Expect token credential authentication is selected over account key or sas token.
+            assertEquals(token_credential_type, mock.storageSettings.get("azure1").getTokenCredentialType());
+            assertEquals(token_credential_type, mock.storageSettings.get("azure2").getTokenCredentialType());
+            assertEquals(token_credential_type, mock.storageSettings.get("azure3").getTokenCredentialType());
+            assertEquals(token_credential_type, mock.storageSettings.get("azure4").getTokenCredentialType());
+        }
+    }
+
+    public void testTokenCredentialWhenAccountIsNotProvided() {
+        // Setting with an account specified
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+
+        // Enabled Managed Identity in the settings
+        final Settings settings = Settings.builder()
+            .setSecureSettings(secureSettings)
+            .put("azure.client.azure.token_credential_type", TokenCredentialType.MANAGED_IDENTITY.name())
+            .build();
+        final Exception e = expectThrows(Exception.class, () -> storageServiceWithSettingsValidation(settings));
+
+        // Expect failure due to missing account name
+        assertEquals(
+            "missing required setting [azure.client.azure.account] for setting [azure.client.azure.token_credential_type]",
+            e.getMessage()
+        );
+    }
+
+    public void testAuthenticationMethodNotProvided() {
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        // Azure client without account key and sas token.
+        secureSettings.setString("azure.client.azure.account", "myaccount");
+
+        // Disabled Managed Identity in the settings by default
+        final Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
+        final SettingsException e = expectThrows(SettingsException.class, () -> storageServiceWithSettingsValidation(settings));
+
+        // Expect fall back to authentication via sas token or account key when token credential is not specified.
+        assertEquals("Neither a secret key nor a shared access token was set.", e.getMessage());
+    }
+
+    public void testSettingTokenCredentialTypeToBeEmpty() {
+        // Azure clients without account key and sas token.
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        secureSettings.setString("azure.client.azure1.account", "myaccount");
+        final Settings settings = Settings.builder()
+            .setSecureSettings(secureSettings)
+            .put("azure.client.azure1.token_credential_type", "")
+            .build();
+        // Expect fall back to authentication via sas token or account key when token credential is not specified.
+        final SettingsException e = expectThrows(SettingsException.class, () -> storageServiceWithSettingsValidation(settings));
+        assertEquals("Neither a secret key nor a shared access token was set.", e.getMessage());
+
+        // Azure clients without account key and sas token.
+        final MockSecureSettings secureSettings2 = new MockSecureSettings();
+        secureSettings2.setString("azure.client.azure2.account", "myaccount");
+        final Settings settings2 = Settings.builder()
+            .setSecureSettings(secureSettings2)
+            .put("azure.client.azure2.token_credential_type", "x")
+            .build();
+        // Expect failing token credential type checks
+        final IllegalArgumentException e2 = expectThrows(
+            IllegalArgumentException.class,
+            () -> storageServiceWithSettingsValidation(settings2)
+        );
+        assertEquals(
+            "The token credential type 'x' is unsupported, please use one of the following values: "
+                + String.join(", ", TokenCredentialType.getTokenCredentialTypes()),
+            e2.getMessage()
+        );
+    }
+
+    public void testManagedIdentityIsEnabled() throws IOException {
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        // Azure client without account key or sas token.
+        secureSettings.setString("azure.client.azure.account", "myaccount");
+
+        // Enabled Managed Identity in the settings.
+        final Settings settings = Settings.builder()
+            .setSecureSettings(secureSettings)
+            .put("azure.client.azure.token_credential_type", TokenCredentialType.MANAGED_IDENTITY.name())
+            .build();
+
+        try (final AzureStorageService mock = storageServiceWithSettingsValidation(settings)) {
+            assertEquals(mock.storageSettings.get("azure").getTokenCredentialType(), TokenCredentialType.MANAGED_IDENTITY.name());
+        }
+    }
+
+    public void testNonTokenCredentialAuthenticationEnabled() throws IOException {
+        final MockSecureSettings secureSettings = new MockSecureSettings();
+        // Azure client account key.
+        secureSettings.setString("azure.client.azure1.account", "myaccount1");
+        secureSettings.setString("azure.client.azure1.sas_token", encodeKey("mysastoken"));
+
+        // Azure client with sas token
+        secureSettings.setString("azure.client.azure2.account", "myaccount2");
+        secureSettings.setString("azure.client.azure2.key", encodeKey("mykey"));
+
+        final Settings settings = Settings.builder().setSecureSettings(secureSettings).build();
+        try (final AzureStorageService mock = storageServiceWithSettingsValidation(settings)) {
+            // Expect token credential is not enabled
+            assertEquals(mock.storageSettings.get("azure1").getTokenCredentialType(), "");
+            assertEquals(mock.storageSettings.get("azure2").getTokenCredentialType(), "");
+        }
+    }
+
     private static MockSecureSettings buildSecureSettings() {
         final MockSecureSettings secureSettings = new MockSecureSettings();
         secureSettings.setString("azure.client.azure1.account", "myaccount1");
diff --git a/plugins/repository-gcs/build.gradle b/plugins/repository-gcs/build.gradle
index 1dfc64e19601c..110df89f25de8 100644
--- a/plugins/repository-gcs/build.gradle
+++ b/plugins/repository-gcs/build.gradle
@@ -61,7 +61,7 @@ dependencies {
   api 'com.google.api-client:google-api-client:2.2.0'
 
   api 'com.google.api.grpc:proto-google-common-protos:2.37.1'
-  api 'com.google.api.grpc:proto-google-iam-v1:0.12.0'
+  api 'com.google.api.grpc:proto-google-iam-v1:1.33.0'
 
   api "com.google.auth:google-auth-library-credentials:${versions.google_auth}"
   api "com.google.auth:google-auth-library-oauth2-http:${versions.google_auth}"
@@ -75,10 +75,10 @@ dependencies {
   runtimeOnly "com.google.guava:guava:${versions.guava}"
   api 'com.google.guava:failureaccess:1.0.1'
 
-  api 'com.google.http-client:google-http-client:1.43.3'
-  api 'com.google.http-client:google-http-client-appengine:1.43.3'
-  api 'com.google.http-client:google-http-client-gson:1.43.3'
-  api 'com.google.http-client:google-http-client-jackson2:1.44.1'
+  api "com.google.http-client:google-http-client:${versions.google_http_client}"
+  api "com.google.http-client:google-http-client-appengine:${versions.google_http_client}"
+  api "com.google.http-client:google-http-client-gson:${versions.google_http_client}"
+  api "com.google.http-client:google-http-client-jackson2:${versions.google_http_client}"
 
   api 'com.google.oauth-client:google-oauth-client:1.34.1'
 
diff --git a/plugins/repository-gcs/licenses/commons-codec-1.15.jar.sha1 b/plugins/repository-gcs/licenses/commons-codec-1.15.jar.sha1
deleted file mode 100644
index 62d99837b87e1..0000000000000
--- a/plugins/repository-gcs/licenses/commons-codec-1.15.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-49d94806b6e3dc933dacbd8acb0fdbab8ebd1e5d
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/commons-codec-1.16.1.jar.sha1 b/plugins/repository-gcs/licenses/commons-codec-1.16.1.jar.sha1
new file mode 100644
index 0000000000000..6b8803089c6d7
--- /dev/null
+++ b/plugins/repository-gcs/licenses/commons-codec-1.16.1.jar.sha1
@@ -0,0 +1 @@
+47bd4d333fba53406f6c6c51884ddbca435c8862
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-http-client-1.43.3.jar.sha1 b/plugins/repository-gcs/licenses/google-http-client-1.43.3.jar.sha1
deleted file mode 100644
index 800467de8bdf3..0000000000000
--- a/plugins/repository-gcs/licenses/google-http-client-1.43.3.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-a758b82e55a2f5f681e289c5ed384d3dbda6f3cd
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-http-client-1.44.1.jar.sha1 b/plugins/repository-gcs/licenses/google-http-client-1.44.1.jar.sha1
new file mode 100644
index 0000000000000..501f268254fbc
--- /dev/null
+++ b/plugins/repository-gcs/licenses/google-http-client-1.44.1.jar.sha1
@@ -0,0 +1 @@
+d8956bacb8a4011365fa15a690482c49a70c78c5
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-http-client-appengine-1.43.3.jar.sha1 b/plugins/repository-gcs/licenses/google-http-client-appengine-1.43.3.jar.sha1
deleted file mode 100644
index 4adcca6a55902..0000000000000
--- a/plugins/repository-gcs/licenses/google-http-client-appengine-1.43.3.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-09d6cbdde6ea3469a67601a811b4e83de3e68a79
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-http-client-appengine-1.44.1.jar.sha1 b/plugins/repository-gcs/licenses/google-http-client-appengine-1.44.1.jar.sha1
new file mode 100644
index 0000000000000..7b27b165453cd
--- /dev/null
+++ b/plugins/repository-gcs/licenses/google-http-client-appengine-1.44.1.jar.sha1
@@ -0,0 +1 @@
+da4f9f691edb7a9f00cd806157a4990cb7e07711
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-http-client-gson-1.43.3.jar.sha1 b/plugins/repository-gcs/licenses/google-http-client-gson-1.43.3.jar.sha1
deleted file mode 100644
index 43f4fe4a127e1..0000000000000
--- a/plugins/repository-gcs/licenses/google-http-client-gson-1.43.3.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-252e267acf720ef6333488740a696a1d5e204639
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/google-http-client-gson-1.44.1.jar.sha1 b/plugins/repository-gcs/licenses/google-http-client-gson-1.44.1.jar.sha1
new file mode 100644
index 0000000000000..90ddf3ddc5ee6
--- /dev/null
+++ b/plugins/repository-gcs/licenses/google-http-client-gson-1.44.1.jar.sha1
@@ -0,0 +1 @@
+f3b8967c6f7078da6380687859d0873105f84d39
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/proto-google-iam-v1-0.12.0.jar.sha1 b/plugins/repository-gcs/licenses/proto-google-iam-v1-0.12.0.jar.sha1
deleted file mode 100644
index 2bfae3456d499..0000000000000
--- a/plugins/repository-gcs/licenses/proto-google-iam-v1-0.12.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-ea312c0250a5d0a7cdd1b20bc2c3259938b79855
\ No newline at end of file
diff --git a/plugins/repository-gcs/licenses/proto-google-iam-v1-1.33.0.jar.sha1 b/plugins/repository-gcs/licenses/proto-google-iam-v1-1.33.0.jar.sha1
new file mode 100644
index 0000000000000..ba04056c54697
--- /dev/null
+++ b/plugins/repository-gcs/licenses/proto-google-iam-v1-1.33.0.jar.sha1
@@ -0,0 +1 @@
+4766da92d1f36c8b612c1c142d5f3ace3774f098
\ No newline at end of file
diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle
index c1f94320f2681..e019a878dfcf0 100644
--- a/plugins/repository-hdfs/build.gradle
+++ b/plugins/repository-hdfs/build.gradle
@@ -70,18 +70,18 @@ dependencies {
   api 'com.google.code.gson:gson:2.10.1'
   runtimeOnly "com.google.guava:guava:${versions.guava}"
   api "commons-logging:commons-logging:${versions.commonslogging}"
-  api 'commons-cli:commons-cli:1.6.0'
+  api 'commons-cli:commons-cli:1.7.0'
   api "commons-codec:commons-codec:${versions.commonscodec}"
   api 'commons-collections:commons-collections:3.2.2'
   api "org.apache.commons:commons-compress:${versions.commonscompress}"
-  api 'org.apache.commons:commons-configuration2:2.10.0'
-  api 'commons-io:commons-io:2.15.1'
+  api 'org.apache.commons:commons-configuration2:2.10.1'
+  api "commons-io:commons-io:${versions.commonsio}"
   api 'org.apache.commons:commons-lang3:3.14.0'
   implementation 'com.google.re2j:re2j:1.7'
   api 'javax.servlet:servlet-api:2.5'
   api "org.slf4j:slf4j-api:${versions.slf4j}"
   api "org.apache.logging.log4j:log4j-slf4j-impl:${versions.log4j}"
-  api 'net.minidev:json-smart:2.5.0'
+  api 'net.minidev:json-smart:2.5.1'
   api "io.netty:netty-all:${versions.netty}"
   implementation "com.fasterxml.woodstox:woodstox-core:${versions.woodstox}"
   implementation 'org.codehaus.woodstox:stax2-api:4.2.2'
diff --git a/plugins/repository-hdfs/licenses/commons-cli-1.6.0.jar.sha1 b/plugins/repository-hdfs/licenses/commons-cli-1.6.0.jar.sha1
deleted file mode 100644
index bb94eda6814ea..0000000000000
--- a/plugins/repository-hdfs/licenses/commons-cli-1.6.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-38166a23afb5bd5520f739b87b3be87f7f0fb96d
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/commons-cli-1.7.0.jar.sha1 b/plugins/repository-hdfs/licenses/commons-cli-1.7.0.jar.sha1
new file mode 100644
index 0000000000000..759bc9275d346
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/commons-cli-1.7.0.jar.sha1
@@ -0,0 +1 @@
+6504b3f17e8bc5adc6b6c8deecc90144d0154075
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/commons-codec-1.15.jar.sha1 b/plugins/repository-hdfs/licenses/commons-codec-1.15.jar.sha1
deleted file mode 100644
index 62d99837b87e1..0000000000000
--- a/plugins/repository-hdfs/licenses/commons-codec-1.15.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-49d94806b6e3dc933dacbd8acb0fdbab8ebd1e5d
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/commons-codec-1.16.1.jar.sha1 b/plugins/repository-hdfs/licenses/commons-codec-1.16.1.jar.sha1
new file mode 100644
index 0000000000000..6b8803089c6d7
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/commons-codec-1.16.1.jar.sha1
@@ -0,0 +1 @@
+47bd4d333fba53406f6c6c51884ddbca435c8862
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/commons-compress-1.24.0.jar.sha1 b/plugins/repository-hdfs/licenses/commons-compress-1.24.0.jar.sha1
deleted file mode 100644
index 23999d1bfbde4..0000000000000
--- a/plugins/repository-hdfs/licenses/commons-compress-1.24.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-b4b1b5a3d9573b2970fddab236102c0a4d27d35e
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/commons-compress-1.26.1.jar.sha1 b/plugins/repository-hdfs/licenses/commons-compress-1.26.1.jar.sha1
new file mode 100644
index 0000000000000..912bda85de18a
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/commons-compress-1.26.1.jar.sha1
@@ -0,0 +1 @@
+44331c1130c370e726a2e1a3e6fba6d2558ef04a
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/commons-configuration2-2.10.0.jar.sha1 b/plugins/repository-hdfs/licenses/commons-configuration2-2.10.0.jar.sha1
deleted file mode 100644
index 17d1b64781e5b..0000000000000
--- a/plugins/repository-hdfs/licenses/commons-configuration2-2.10.0.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-2b93eff3c83e5372262ed4996b609336305a810f
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/commons-configuration2-2.10.1.jar.sha1 b/plugins/repository-hdfs/licenses/commons-configuration2-2.10.1.jar.sha1
new file mode 100644
index 0000000000000..d4c0f8417d357
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/commons-configuration2-2.10.1.jar.sha1
@@ -0,0 +1 @@
+2b681b3bcddeaa5bf5c2a2939cd77e2f9ad6efda
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/commons-io-2.15.1.jar.sha1 b/plugins/repository-hdfs/licenses/commons-io-2.15.1.jar.sha1
deleted file mode 100644
index 47c5d13812a36..0000000000000
--- a/plugins/repository-hdfs/licenses/commons-io-2.15.1.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-f11560da189ab563a5c8e351941415430e9304ea
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/commons-io-2.16.0.jar.sha1 b/plugins/repository-hdfs/licenses/commons-io-2.16.0.jar.sha1
new file mode 100644
index 0000000000000..6a7b638719fa3
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/commons-io-2.16.0.jar.sha1
@@ -0,0 +1 @@
+27875a7935f1ddcc13267eb6fae1f719e0409572
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/json-smart-2.5.1.jar.sha1 b/plugins/repository-hdfs/licenses/json-smart-2.5.1.jar.sha1
new file mode 100644
index 0000000000000..fe23968afce1e
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/json-smart-2.5.1.jar.sha1
@@ -0,0 +1 @@
+4c11d2808d009132dfbbf947ebf37de6bf266c8e
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/netty-all-4.1.107.Final.jar.sha1 b/plugins/repository-hdfs/licenses/netty-all-4.1.107.Final.jar.sha1
deleted file mode 100644
index 0e3595fecb0d2..0000000000000
--- a/plugins/repository-hdfs/licenses/netty-all-4.1.107.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3019703b67413ef3d6150da1f49753f4010507ce
\ No newline at end of file
diff --git a/plugins/repository-hdfs/licenses/netty-all-4.1.109.Final.jar.sha1 b/plugins/repository-hdfs/licenses/netty-all-4.1.109.Final.jar.sha1
new file mode 100644
index 0000000000000..a874755cc29da
--- /dev/null
+++ b/plugins/repository-hdfs/licenses/netty-all-4.1.109.Final.jar.sha1
@@ -0,0 +1 @@
+3ba1acc8ff088334f2ac5556663f8b737eb8b571
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/commons-codec-1.15.jar.sha1 b/plugins/repository-s3/licenses/commons-codec-1.15.jar.sha1
deleted file mode 100644
index 62d99837b87e1..0000000000000
--- a/plugins/repository-s3/licenses/commons-codec-1.15.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-49d94806b6e3dc933dacbd8acb0fdbab8ebd1e5d
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/commons-codec-1.16.1.jar.sha1 b/plugins/repository-s3/licenses/commons-codec-1.16.1.jar.sha1
new file mode 100644
index 0000000000000..6b8803089c6d7
--- /dev/null
+++ b/plugins/repository-s3/licenses/commons-codec-1.16.1.jar.sha1
@@ -0,0 +1 @@
+47bd4d333fba53406f6c6c51884ddbca435c8862
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-buffer-4.1.107.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-buffer-4.1.107.Final.jar.sha1
deleted file mode 100644
index beb44fc0f4cf9..0000000000000
--- a/plugins/repository-s3/licenses/netty-buffer-4.1.107.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-8509a72b8a5a2d33d611e99254aed39765c3ad82
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-buffer-4.1.109.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-buffer-4.1.109.Final.jar.sha1
new file mode 100644
index 0000000000000..76b51cdae3867
--- /dev/null
+++ b/plugins/repository-s3/licenses/netty-buffer-4.1.109.Final.jar.sha1
@@ -0,0 +1 @@
+9d21d602ad7c639fa16b1d26559065d310a34c51
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-codec-4.1.107.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-4.1.107.Final.jar.sha1
deleted file mode 100644
index 4c74bb06fd83b..0000000000000
--- a/plugins/repository-s3/licenses/netty-codec-4.1.107.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-0a1d32debf2ed07c5852ab5b2904c43adb76c39e
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-codec-4.1.109.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-4.1.109.Final.jar.sha1
new file mode 100644
index 0000000000000..1bccee872152d
--- /dev/null
+++ b/plugins/repository-s3/licenses/netty-codec-4.1.109.Final.jar.sha1
@@ -0,0 +1 @@
+16e0b2beb49318a549d3ba5d66d707bd5daa8c97
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-codec-http-4.1.107.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-http-4.1.107.Final.jar.sha1
deleted file mode 100644
index 38eb2e5bad80a..0000000000000
--- a/plugins/repository-s3/licenses/netty-codec-http-4.1.107.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-04d8e9e51b7254bd26a42fe17bdcae32e4c6ebb3
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-codec-http-4.1.109.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-http-4.1.109.Final.jar.sha1
new file mode 100644
index 0000000000000..3423fb94e8497
--- /dev/null
+++ b/plugins/repository-s3/licenses/netty-codec-http-4.1.109.Final.jar.sha1
@@ -0,0 +1 @@
+6dca43cedc0b2dc6bf57bdc85fce6ffca3e6b72a
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-codec-http2-4.1.107.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-http2-4.1.107.Final.jar.sha1
deleted file mode 100644
index 5b3d3311edc9f..0000000000000
--- a/plugins/repository-s3/licenses/netty-codec-http2-4.1.107.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3885ffe7dd05c9773df70c61009f34a5a8a383ec
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-codec-http2-4.1.109.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-codec-http2-4.1.109.Final.jar.sha1
new file mode 100644
index 0000000000000..b83ad36222d07
--- /dev/null
+++ b/plugins/repository-s3/licenses/netty-codec-http2-4.1.109.Final.jar.sha1
@@ -0,0 +1 @@
+6bd4a54b69a81356393f6e4621bad40754f8a5a2
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-common-4.1.107.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-common-4.1.107.Final.jar.sha1
deleted file mode 100644
index bbe91c6ccfb1d..0000000000000
--- a/plugins/repository-s3/licenses/netty-common-4.1.107.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-4f17a547530d64becd7179507b25f4154bcfba57
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-common-4.1.109.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-common-4.1.109.Final.jar.sha1
new file mode 100644
index 0000000000000..5172500557f8b
--- /dev/null
+++ b/plugins/repository-s3/licenses/netty-common-4.1.109.Final.jar.sha1
@@ -0,0 +1 @@
+da63e54ee1ca69abf4206cb74fadef7f50850911
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-handler-4.1.107.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-handler-4.1.107.Final.jar.sha1
deleted file mode 100644
index ba27b38632622..0000000000000
--- a/plugins/repository-s3/licenses/netty-handler-4.1.107.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d4c6b05f4d9aca117981297fb7f02953102ebb5e
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-handler-4.1.109.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-handler-4.1.109.Final.jar.sha1
new file mode 100644
index 0000000000000..cabe61b300523
--- /dev/null
+++ b/plugins/repository-s3/licenses/netty-handler-4.1.109.Final.jar.sha1
@@ -0,0 +1 @@
+9167863307b3c44cc12262e7b5512de3499b9c4a
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-resolver-4.1.107.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-resolver-4.1.107.Final.jar.sha1
deleted file mode 100644
index 3bc0f7b3fed09..0000000000000
--- a/plugins/repository-s3/licenses/netty-resolver-4.1.107.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-dfee84308341a42131dd0f8ac0e1e02d627c19f3
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-resolver-4.1.109.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-resolver-4.1.109.Final.jar.sha1
new file mode 100644
index 0000000000000..14e21cc0cdb60
--- /dev/null
+++ b/plugins/repository-s3/licenses/netty-resolver-4.1.109.Final.jar.sha1
@@ -0,0 +1 @@
+055485ac976e27c8bb67ee111a8490c58f67b70c
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-transport-4.1.107.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-4.1.107.Final.jar.sha1
deleted file mode 100644
index 19419999300dd..0000000000000
--- a/plugins/repository-s3/licenses/netty-transport-4.1.107.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d6a105c621b47d1410e0e09419d7209d2d46e914
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-transport-4.1.109.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-4.1.109.Final.jar.sha1
new file mode 100644
index 0000000000000..6b23d0883e31f
--- /dev/null
+++ b/plugins/repository-s3/licenses/netty-transport-4.1.109.Final.jar.sha1
@@ -0,0 +1 @@
+79e3b07d58ef03c7a860d48f932b720675aa8bd3
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.107.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.107.Final.jar.sha1
deleted file mode 100644
index 6b9a35acb2c20..0000000000000
--- a/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.107.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-9234407d6a46745599735765c4d3755c7fc84162
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.109.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.109.Final.jar.sha1
new file mode 100644
index 0000000000000..83fc39246ef0a
--- /dev/null
+++ b/plugins/repository-s3/licenses/netty-transport-classes-epoll-4.1.109.Final.jar.sha1
@@ -0,0 +1 @@
+7307c8acbc9b331fce3496750a5112bdc726fd2a
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.107.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.107.Final.jar.sha1
deleted file mode 100644
index 407ecaffdad30..0000000000000
--- a/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.107.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-4d61d4959741109b3eccd7337f11fc89fa90a74a
\ No newline at end of file
diff --git a/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.109.Final.jar.sha1 b/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.109.Final.jar.sha1
new file mode 100644
index 0000000000000..5afeb9627c9b5
--- /dev/null
+++ b/plugins/repository-s3/licenses/netty-transport-native-unix-common-4.1.109.Final.jar.sha1
@@ -0,0 +1 @@
+da7fe1e6943cbab8ee48df2beadc2c8304f347a2
\ No newline at end of file
diff --git a/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java b/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java
index da2c6e8c1b0ee..21184380d54a9 100644
--- a/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java
+++ b/plugins/repository-s3/src/internalClusterTest/java/org/opensearch/repositories/s3/S3BlobStoreRepositoryTests.java
@@ -249,7 +249,22 @@ protected S3Repository createRepository(
             ClusterService clusterService,
             RecoverySettings recoverySettings
         ) {
-            return new S3Repository(metadata, registry, service, clusterService, recoverySettings, null, null, null, null, null, false) {
+            return new S3Repository(
+                metadata,
+                registry,
+                service,
+                clusterService,
+                recoverySettings,
+                null,
+                null,
+                null,
+                null,
+                null,
+                false,
+                null,
+                null,
+                null
+            ) {
 
                 @Override
                 public BlobStore blobStore() {
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/GenericStatsMetricPublisher.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/GenericStatsMetricPublisher.java
new file mode 100644
index 0000000000000..136fd68223354
--- /dev/null
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/GenericStatsMetricPublisher.java
@@ -0,0 +1,90 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.repositories.s3;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicLong;
+
+/**
+ * Generic stats of repository-s3 plugin.
+ */
+public class GenericStatsMetricPublisher {
+
+    private final AtomicLong normalPriorityQSize = new AtomicLong();
+    private final AtomicInteger normalPriorityPermits = new AtomicInteger();
+    private final AtomicLong lowPriorityQSize = new AtomicLong();
+    private final AtomicInteger lowPriorityPermits = new AtomicInteger();
+    private final long normalPriorityQCapacity;
+    private final int maxNormalPriorityPermits;
+    private final long lowPriorityQCapacity;
+    private final int maxLowPriorityPermits;
+
+    public GenericStatsMetricPublisher(
+        long normalPriorityQCapacity,
+        int maxNormalPriorityPermits,
+        long lowPriorityQCapacity,
+        int maxLowPriorityPermits
+    ) {
+        this.normalPriorityQCapacity = normalPriorityQCapacity;
+        this.maxNormalPriorityPermits = maxNormalPriorityPermits;
+        this.lowPriorityQCapacity = lowPriorityQCapacity;
+        this.maxLowPriorityPermits = maxLowPriorityPermits;
+    }
+
+    public void updateNormalPriorityQSize(long qSize) {
+        normalPriorityQSize.addAndGet(qSize);
+    }
+
+    public void updateLowPriorityQSize(long qSize) {
+        lowPriorityQSize.addAndGet(qSize);
+    }
+
+    public void updateNormalPermits(boolean increment) {
+        if (increment) {
+            normalPriorityPermits.incrementAndGet();
+        } else {
+            normalPriorityPermits.decrementAndGet();
+        }
+    }
+
+    public void updateLowPermits(boolean increment) {
+        if (increment) {
+            lowPriorityPermits.incrementAndGet();
+        } else {
+            lowPriorityPermits.decrementAndGet();
+        }
+    }
+
+    public long getNormalPriorityQSize() {
+        return normalPriorityQSize.get();
+    }
+
+    public int getAcquiredNormalPriorityPermits() {
+        return normalPriorityPermits.get();
+    }
+
+    public long getLowPriorityQSize() {
+        return lowPriorityQSize.get();
+    }
+
+    public int getAcquiredLowPriorityPermits() {
+        return lowPriorityPermits.get();
+    }
+
+    Map<String, Long> stats() {
+        final Map<String, Long> results = new HashMap<>();
+        results.put("NormalPriorityQUtilization", (normalPriorityQSize.get() * 100) / normalPriorityQCapacity);
+        results.put("LowPriorityQUtilization", (lowPriorityQSize.get() * 100) / lowPriorityQCapacity);
+        results.put("NormalPriorityPermitsUtilization", (normalPriorityPermits.get() * 100L) / maxNormalPriorityPermits);
+        results.put("LowPriorityPermitsUtilization", (lowPriorityPermits.get() * 100L) / maxLowPriorityPermits);
+        return results;
+    }
+}
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java
index 25f361b40636e..acf0c5e83a17b 100644
--- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobContainer.java
@@ -62,6 +62,7 @@
 import software.amazon.awssdk.services.s3.model.UploadPartRequest;
 import software.amazon.awssdk.services.s3.model.UploadPartResponse;
 import software.amazon.awssdk.services.s3.paginators.ListObjectsV2Iterable;
+import software.amazon.awssdk.utils.CollectionUtils;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -77,6 +78,7 @@
 import org.opensearch.common.blobstore.BlobPath;
 import org.opensearch.common.blobstore.BlobStoreException;
 import org.opensearch.common.blobstore.DeleteResult;
+import org.opensearch.common.blobstore.FetchBlobResult;
 import org.opensearch.common.blobstore.stream.read.ReadContext;
 import org.opensearch.common.blobstore.stream.write.WriteContext;
 import org.opensearch.common.blobstore.stream.write.WritePriority;
@@ -88,6 +90,7 @@
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.unit.ByteSizeUnit;
 import org.opensearch.core.common.unit.ByteSizeValue;
+import org.opensearch.repositories.s3.async.SizeBasedBlockingQ;
 import org.opensearch.repositories.s3.async.UploadRequest;
 import org.opensearch.repositories.s3.utils.HttpRangeUtils;
 
@@ -138,6 +141,13 @@ public boolean blobExists(String blobName) {
         }
     }
 
+    @ExperimentalApi
+    @Override
+    public FetchBlobResult readBlobWithMetadata(String blobName) throws IOException {
+        S3RetryingInputStream s3RetryingInputStream = new S3RetryingInputStream(blobStore, buildKey(blobName));
+        return new FetchBlobResult(s3RetryingInputStream, s3RetryingInputStream.getMetadata());
+    }
+
     @Override
     public InputStream readBlob(String blobName) throws IOException {
         return new S3RetryingInputStream(blobStore, buildKey(blobName));
@@ -169,12 +179,27 @@ public long readBlobPreferredLength() {
      */
     @Override
     public void writeBlob(String blobName, InputStream inputStream, long blobSize, boolean failIfAlreadyExists) throws IOException {
+        writeBlobWithMetadata(blobName, inputStream, blobSize, failIfAlreadyExists, null);
+    }
+
+    /**
+     * Write blob with its object metadata.
+     */
+    @ExperimentalApi
+    @Override
+    public void writeBlobWithMetadata(
+        String blobName,
+        InputStream inputStream,
+        long blobSize,
+        boolean failIfAlreadyExists,
+        @Nullable Map<String, String> metadata
+    ) throws IOException {
         assert inputStream.markSupported() : "No mark support on inputStream breaks the S3 SDK's ability to retry requests";
         SocketAccess.doPrivilegedIOException(() -> {
             if (blobSize <= getLargeBlobThresholdInBytes()) {
-                executeSingleUpload(blobStore, buildKey(blobName), inputStream, blobSize);
+                executeSingleUpload(blobStore, buildKey(blobName), inputStream, blobSize, metadata);
             } else {
-                executeMultipartUpload(blobStore, buildKey(blobName), inputStream, blobSize);
+                executeMultipartUpload(blobStore, buildKey(blobName), inputStream, blobSize, metadata);
             }
             return null;
         });
@@ -190,10 +215,18 @@ public void asyncBlobUpload(WriteContext writeContext, ActionListener<Void> comp
             writeContext.getUploadFinalizer(),
             writeContext.doRemoteDataIntegrityCheck(),
             writeContext.getExpectedChecksum(),
-            blobStore.isUploadRetryEnabled()
+            blobStore.isUploadRetryEnabled(),
+            writeContext.getMetadata()
         );
         try {
-            if (uploadRequest.getContentLength() > ByteSizeUnit.GB.toBytes(10) && blobStore.isRedirectLargeUploads()) {
+            // If file size is greater than the queue capacity than SizeBasedBlockingQ will always reject the upload.
+            // Therefore, redirecting it to slow client.
+            if ((uploadRequest.getWritePriority() == WritePriority.LOW
+                && blobStore.getLowPrioritySizeBasedBlockingQ().isMaxCapacityBelowContentLength(uploadRequest.getContentLength()) == false)
+                || (uploadRequest.getWritePriority() != WritePriority.HIGH
+                    && uploadRequest.getWritePriority() != WritePriority.URGENT
+                    && blobStore.getNormalPrioritySizeBasedBlockingQ()
+                        .isMaxCapacityBelowContentLength(uploadRequest.getContentLength()) == false)) {
                 StreamContext streamContext = SocketAccess.doPrivileged(
                     () -> writeContext.getStreamProvider(uploadRequest.getContentLength())
                 );
@@ -203,7 +236,8 @@ public void asyncBlobUpload(WriteContext writeContext, ActionListener<Void> comp
                         blobStore,
                         uploadRequest.getKey(),
                         inputStream.getInputStream(),
-                        uploadRequest.getContentLength()
+                        uploadRequest.getContentLength(),
+                        uploadRequest.getMetadata()
                     );
                     completionListener.onResponse(null);
                 } catch (Exception ex) {
@@ -232,16 +266,30 @@ public void asyncBlobUpload(WriteContext writeContext, ActionListener<Void> comp
                 } else {
                     s3AsyncClient = amazonS3Reference.get().client();
                 }
-                CompletableFuture<Void> completableFuture = blobStore.getAsyncTransferManager()
-                    .uploadObject(s3AsyncClient, uploadRequest, streamContext, blobStore.getStatsMetricPublisher());
-                completableFuture.whenComplete((response, throwable) -> {
-                    if (throwable == null) {
-                        completionListener.onResponse(response);
-                    } else {
-                        Exception ex = throwable instanceof Error ? new Exception(throwable) : (Exception) throwable;
-                        completionListener.onFailure(ex);
-                    }
-                });
+
+                if (writeContext.getWritePriority() == WritePriority.URGENT
+                    || writeContext.getWritePriority() == WritePriority.HIGH
+                    || blobStore.isPermitBackedTransferEnabled() == false) {
+                    createFileCompletableFuture(s3AsyncClient, uploadRequest, streamContext, completionListener);
+                } else if (writeContext.getWritePriority() == WritePriority.LOW) {
+                    blobStore.getLowPrioritySizeBasedBlockingQ()
+                        .produce(
+                            new SizeBasedBlockingQ.Item(
+                                writeContext.getFileSize(),
+                                () -> createFileCompletableFuture(s3AsyncClient, uploadRequest, streamContext, completionListener)
+                            )
+                        );
+                } else if (writeContext.getWritePriority() == WritePriority.NORMAL) {
+                    blobStore.getNormalPrioritySizeBasedBlockingQ()
+                        .produce(
+                            new SizeBasedBlockingQ.Item(
+                                writeContext.getFileSize(),
+                                () -> createFileCompletableFuture(s3AsyncClient, uploadRequest, streamContext, completionListener)
+                            )
+                        );
+                } else {
+                    throw new IllegalStateException("Cannot perform upload for other priority types.");
+                }
             }
         } catch (Exception e) {
             logger.info("exception error from blob container for file {}", writeContext.getFileName());
@@ -249,6 +297,24 @@ public void asyncBlobUpload(WriteContext writeContext, ActionListener<Void> comp
         }
     }
 
+    private CompletableFuture<Void> createFileCompletableFuture(
+        S3AsyncClient s3AsyncClient,
+        UploadRequest uploadRequest,
+        StreamContext streamContext,
+        ActionListener<Void> completionListener
+    ) {
+        CompletableFuture<Void> completableFuture = blobStore.getAsyncTransferManager()
+            .uploadObject(s3AsyncClient, uploadRequest, streamContext, blobStore.getStatsMetricPublisher());
+        return completableFuture.whenComplete((response, throwable) -> {
+            if (throwable == null) {
+                completionListener.onResponse(response);
+            } else {
+                Exception ex = throwable instanceof Error ? new Exception(throwable) : (Exception) throwable;
+                completionListener.onFailure(ex);
+            }
+        });
+    }
+
     @ExperimentalApi
     @Override
     public void readBlobAsync(String blobName, ActionListener<ReadContext> listener) {
@@ -285,7 +351,7 @@ public void readBlobAsync(String blobName, ActionListener<ReadContext> listener)
                             );
                         }
                     }
-                    listener.onResponse(new ReadContext(blobSize, blobPartInputStreamFutures, blobChecksum));
+                    listener.onResponse(new ReadContext.Builder(blobSize, blobPartInputStreamFutures).blobChecksum(blobChecksum).build());
                 } catch (Exception ex) {
                     listener.onFailure(ex);
                 }
@@ -542,8 +608,13 @@ private String buildKey(String blobName) {
     /**
      * Uploads a blob using a single upload request
      */
-    void executeSingleUpload(final S3BlobStore blobStore, final String blobName, final InputStream input, final long blobSize)
-        throws IOException {
+    void executeSingleUpload(
+        final S3BlobStore blobStore,
+        final String blobName,
+        final InputStream input,
+        final long blobSize,
+        final Map<String, String> metadata
+    ) throws IOException {
 
         // Extra safety checks
         if (blobSize > MAX_FILE_SIZE.getBytes()) {
@@ -560,6 +631,10 @@ void executeSingleUpload(final S3BlobStore blobStore, final String blobName, fin
             .storageClass(blobStore.getStorageClass())
             .acl(blobStore.getCannedACL())
             .overrideConfiguration(o -> o.addMetricPublisher(blobStore.getStatsMetricPublisher().putObjectMetricPublisher));
+
+        if (CollectionUtils.isNotEmpty(metadata)) {
+            putObjectRequestBuilder = putObjectRequestBuilder.metadata(metadata);
+        }
         if (blobStore.serverSideEncryption()) {
             putObjectRequestBuilder.serverSideEncryption(ServerSideEncryption.AES256);
         }
@@ -583,8 +658,13 @@ void executeSingleUpload(final S3BlobStore blobStore, final String blobName, fin
     /**
      * Uploads a blob using multipart upload requests.
      */
-    void executeMultipartUpload(final S3BlobStore blobStore, final String blobName, final InputStream input, final long blobSize)
-        throws IOException {
+    void executeMultipartUpload(
+        final S3BlobStore blobStore,
+        final String blobName,
+        final InputStream input,
+        final long blobSize,
+        final Map<String, String> metadata
+    ) throws IOException {
 
         ensureMultiPartUploadSize(blobSize);
         final long partSize = blobStore.bufferSizeInBytes();
@@ -609,6 +689,10 @@ void executeMultipartUpload(final S3BlobStore blobStore, final String blobName,
             .acl(blobStore.getCannedACL())
             .overrideConfiguration(o -> o.addMetricPublisher(blobStore.getStatsMetricPublisher().multipartUploadMetricCollector));
 
+        if (CollectionUtils.isNotEmpty(metadata)) {
+            createMultipartUploadRequestBuilder.metadata(metadata);
+        }
+
         if (blobStore.serverSideEncryption()) {
             createMultipartUploadRequestBuilder.serverSideEncryption(ServerSideEncryption.AES256);
         }
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java
index fc70fbb0db00e..de815f9202f44 100644
--- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3BlobStore.java
@@ -45,6 +45,7 @@
 import org.opensearch.core.common.unit.ByteSizeValue;
 import org.opensearch.repositories.s3.async.AsyncExecutorContainer;
 import org.opensearch.repositories.s3.async.AsyncTransferManager;
+import org.opensearch.repositories.s3.async.SizeBasedBlockingQ;
 
 import java.io.IOException;
 import java.util.Collections;
@@ -56,6 +57,7 @@
 import static org.opensearch.repositories.s3.S3Repository.BUFFER_SIZE_SETTING;
 import static org.opensearch.repositories.s3.S3Repository.BULK_DELETE_SIZE;
 import static org.opensearch.repositories.s3.S3Repository.CANNED_ACL_SETTING;
+import static org.opensearch.repositories.s3.S3Repository.PERMIT_BACKED_TRANSFER_ENABLED;
 import static org.opensearch.repositories.s3.S3Repository.REDIRECT_LARGE_S3_UPLOAD;
 import static org.opensearch.repositories.s3.S3Repository.SERVER_SIDE_ENCRYPTION_SETTING;
 import static org.opensearch.repositories.s3.S3Repository.STORAGE_CLASS_SETTING;
@@ -77,6 +79,8 @@ class S3BlobStore implements BlobStore {
 
     private volatile boolean uploadRetryEnabled;
 
+    private volatile boolean permitBackedTransferEnabled;
+
     private volatile boolean serverSideEncryption;
 
     private volatile ObjectCannedACL cannedACL;
@@ -94,6 +98,9 @@ class S3BlobStore implements BlobStore {
     private final AsyncExecutorContainer priorityExecutorBuilder;
     private final AsyncExecutorContainer normalExecutorBuilder;
     private final boolean multipartUploadEnabled;
+    private final SizeBasedBlockingQ normalPrioritySizeBasedBlockingQ;
+    private final SizeBasedBlockingQ lowPrioritySizeBasedBlockingQ;
+    private final GenericStatsMetricPublisher genericStatsMetricPublisher;
 
     S3BlobStore(
         S3Service service,
@@ -109,7 +116,10 @@ class S3BlobStore implements BlobStore {
         AsyncTransferManager asyncTransferManager,
         AsyncExecutorContainer urgentExecutorBuilder,
         AsyncExecutorContainer priorityExecutorBuilder,
-        AsyncExecutorContainer normalExecutorBuilder
+        AsyncExecutorContainer normalExecutorBuilder,
+        SizeBasedBlockingQ normalPrioritySizeBasedBlockingQ,
+        SizeBasedBlockingQ lowPrioritySizeBasedBlockingQ,
+        GenericStatsMetricPublisher genericStatsMetricPublisher
     ) {
         this.service = service;
         this.s3AsyncService = s3AsyncService;
@@ -128,6 +138,10 @@ class S3BlobStore implements BlobStore {
         // Settings to initialize blobstore with.
         this.redirectLargeUploads = REDIRECT_LARGE_S3_UPLOAD.get(repositoryMetadata.settings());
         this.uploadRetryEnabled = UPLOAD_RETRY_ENABLED.get(repositoryMetadata.settings());
+        this.normalPrioritySizeBasedBlockingQ = normalPrioritySizeBasedBlockingQ;
+        this.lowPrioritySizeBasedBlockingQ = lowPrioritySizeBasedBlockingQ;
+        this.genericStatsMetricPublisher = genericStatsMetricPublisher;
+        this.permitBackedTransferEnabled = PERMIT_BACKED_TRANSFER_ENABLED.get(repositoryMetadata.settings());
     }
 
     @Override
@@ -141,6 +155,7 @@ public void reload(RepositoryMetadata repositoryMetadata) {
         this.bulkDeletesSize = BULK_DELETE_SIZE.get(repositoryMetadata.settings());
         this.redirectLargeUploads = REDIRECT_LARGE_S3_UPLOAD.get(repositoryMetadata.settings());
         this.uploadRetryEnabled = UPLOAD_RETRY_ENABLED.get(repositoryMetadata.settings());
+        this.permitBackedTransferEnabled = PERMIT_BACKED_TRANSFER_ENABLED.get(repositoryMetadata.settings());
     }
 
     @Override
@@ -168,6 +183,10 @@ public boolean isUploadRetryEnabled() {
         return uploadRetryEnabled;
     }
 
+    public boolean isPermitBackedTransferEnabled() {
+        return permitBackedTransferEnabled;
+    }
+
     public String bucket() {
         return bucket;
     }
@@ -184,6 +203,14 @@ public int getBulkDeletesSize() {
         return bulkDeletesSize;
     }
 
+    public SizeBasedBlockingQ getNormalPrioritySizeBasedBlockingQ() {
+        return normalPrioritySizeBasedBlockingQ;
+    }
+
+    public SizeBasedBlockingQ getLowPrioritySizeBasedBlockingQ() {
+        return lowPrioritySizeBasedBlockingQ;
+    }
+
     @Override
     public BlobContainer blobContainer(BlobPath path) {
         return new S3BlobContainer(path, this);
@@ -201,7 +228,9 @@ public void close() throws IOException {
 
     @Override
     public Map<String, Long> stats() {
-        return statsMetricPublisher.getStats().toMap();
+        Map<String, Long> stats = statsMetricPublisher.getStats().toMap();
+        stats.putAll(genericStatsMetricPublisher.stats());
+        return stats;
     }
 
     @Override
@@ -211,6 +240,7 @@ public Map<Metric, Map<String, Long>> extendedStats() {
         }
         Map<Metric, Map<String, Long>> extendedStats = new HashMap<>();
         statsMetricPublisher.getExtendedStats().forEach((k, v) -> extendedStats.put(k, v.toMap()));
+        extendedStats.put(Metric.GENERIC_STATS, genericStatsMetricPublisher.stats());
         return extendedStats;
     }
 
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java
index f7772a57c9afd..01b75c0b915f2 100644
--- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3Repository.java
@@ -49,6 +49,7 @@
 import org.opensearch.common.settings.SecureSetting;
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.concurrent.OpenSearchExecutors;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.settings.SecureString;
@@ -63,6 +64,7 @@
 import org.opensearch.repositories.blobstore.MeteredBlobStoreRepository;
 import org.opensearch.repositories.s3.async.AsyncExecutorContainer;
 import org.opensearch.repositories.s3.async.AsyncTransferManager;
+import org.opensearch.repositories.s3.async.SizeBasedBlockingQ;
 import org.opensearch.snapshots.SnapshotId;
 import org.opensearch.snapshots.SnapshotInfo;
 import org.opensearch.threadpool.Scheduler;
@@ -156,6 +158,15 @@ class S3Repository extends MeteredBlobStoreRepository {
         Setting.Property.NodeScope
     );
 
+    /**
+     * Whether large uploads need to be redirected to slow sync s3 client.
+     */
+    static final Setting<Boolean> PERMIT_BACKED_TRANSFER_ENABLED = Setting.boolSetting(
+        "permit_backed_transfer_enabled",
+        true,
+        Setting.Property.NodeScope
+    );
+
     /**
      * Whether retry on uploads are enabled. This setting wraps inputstream with buffered stream to enable retries.
      */
@@ -193,6 +204,37 @@ class S3Repository extends MeteredBlobStoreRepository {
         true,
         Setting.Property.NodeScope
     );
+    /**
+     * Percentage of total available permits to be available for priority transfers.
+     */
+    public static Setting<Integer> S3_PRIORITY_PERMIT_ALLOCATION_PERCENT = Setting.intSetting(
+        "s3_priority_permit_alloc_perc",
+        70,
+        21,
+        80,
+        Setting.Property.NodeScope
+    );
+
+    /**
+     * Duration in minutes to wait for a permit in case no permit is available.
+     */
+    public static Setting<Integer> S3_PERMIT_WAIT_DURATION_MIN = Setting.intSetting(
+        "s3_permit_wait_duration_min",
+        5,
+        1,
+        10,
+        Setting.Property.NodeScope
+    );
+
+    /**
+     * Number of transfer queue consumers
+     */
+    public static Setting<Integer> S3_TRANSFER_QUEUE_CONSUMERS = new Setting<>(
+        "s3_transfer_queue_consumers",
+        (s) -> Integer.toString(Math.max(5, OpenSearchExecutors.allocatedProcessors(s) * 2)),
+        (s) -> Setting.parseInt(s, 5, "s3_transfer_queue_consumers"),
+        Setting.Property.NodeScope
+    );
 
     /**
      * Big files can be broken down into chunks during snapshotting if needed. Defaults to 1g.
@@ -252,6 +294,9 @@ class S3Repository extends MeteredBlobStoreRepository {
     private final AsyncExecutorContainer priorityExecutorBuilder;
     private final AsyncExecutorContainer normalExecutorBuilder;
     private final Path pluginConfigPath;
+    private final SizeBasedBlockingQ normalPrioritySizeBasedBlockingQ;
+    private final SizeBasedBlockingQ lowPrioritySizeBasedBlockingQ;
+    private final GenericStatsMetricPublisher genericStatsMetricPublisher;
 
     private volatile int bulkDeletesSize;
 
@@ -267,7 +312,10 @@ class S3Repository extends MeteredBlobStoreRepository {
         final AsyncExecutorContainer priorityExecutorBuilder,
         final AsyncExecutorContainer normalExecutorBuilder,
         final S3AsyncService s3AsyncService,
-        final boolean multipartUploadEnabled
+        final boolean multipartUploadEnabled,
+        final SizeBasedBlockingQ normalPrioritySizeBasedBlockingQ,
+        final SizeBasedBlockingQ lowPrioritySizeBasedBlockingQ,
+        final GenericStatsMetricPublisher genericStatsMetricPublisher
     ) {
         this(
             metadata,
@@ -281,7 +329,10 @@ class S3Repository extends MeteredBlobStoreRepository {
             normalExecutorBuilder,
             s3AsyncService,
             multipartUploadEnabled,
-            Path.of("")
+            Path.of(""),
+            normalPrioritySizeBasedBlockingQ,
+            lowPrioritySizeBasedBlockingQ,
+            genericStatsMetricPublisher
         );
     }
 
@@ -300,7 +351,10 @@ class S3Repository extends MeteredBlobStoreRepository {
         final AsyncExecutorContainer normalExecutorBuilder,
         final S3AsyncService s3AsyncService,
         final boolean multipartUploadEnabled,
-        Path pluginConfigPath
+        Path pluginConfigPath,
+        final SizeBasedBlockingQ normalPrioritySizeBasedBlockingQ,
+        final SizeBasedBlockingQ lowPrioritySizeBasedBlockingQ,
+        final GenericStatsMetricPublisher genericStatsMetricPublisher
     ) {
         super(metadata, namedXContentRegistry, clusterService, recoverySettings, buildLocation(metadata));
         this.service = service;
@@ -311,6 +365,9 @@ class S3Repository extends MeteredBlobStoreRepository {
         this.urgentExecutorBuilder = urgentExecutorBuilder;
         this.priorityExecutorBuilder = priorityExecutorBuilder;
         this.normalExecutorBuilder = normalExecutorBuilder;
+        this.normalPrioritySizeBasedBlockingQ = normalPrioritySizeBasedBlockingQ;
+        this.lowPrioritySizeBasedBlockingQ = lowPrioritySizeBasedBlockingQ;
+        this.genericStatsMetricPublisher = genericStatsMetricPublisher;
 
         validateRepositoryMetadata(metadata);
         readRepositoryMetadata();
@@ -373,7 +430,10 @@ protected S3BlobStore createBlobStore() {
             asyncUploadUtils,
             urgentExecutorBuilder,
             priorityExecutorBuilder,
-            normalExecutorBuilder
+            normalExecutorBuilder,
+            normalPrioritySizeBasedBlockingQ,
+            lowPrioritySizeBasedBlockingQ,
+            genericStatsMetricPublisher
         );
     }
 
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java
index e7d2a4d024e60..110d91bfbd822 100644
--- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java
@@ -41,6 +41,9 @@
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.util.concurrent.OpenSearchExecutors;
 import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
+import org.opensearch.core.common.util.CollectionUtils;
 import org.opensearch.core.xcontent.NamedXContentRegistry;
 import org.opensearch.env.Environment;
 import org.opensearch.env.NodeEnvironment;
@@ -53,6 +56,8 @@
 import org.opensearch.repositories.s3.async.AsyncExecutorContainer;
 import org.opensearch.repositories.s3.async.AsyncTransferEventLoopGroup;
 import org.opensearch.repositories.s3.async.AsyncTransferManager;
+import org.opensearch.repositories.s3.async.SizeBasedBlockingQ;
+import org.opensearch.repositories.s3.async.TransferSemaphoresHolder;
 import org.opensearch.script.ScriptService;
 import org.opensearch.threadpool.ExecutorBuilder;
 import org.opensearch.threadpool.FixedExecutorBuilder;
@@ -69,6 +74,8 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
 import java.util.function.Supplier;
 
 /**
@@ -82,6 +89,8 @@ public class S3RepositoryPlugin extends Plugin implements RepositoryPlugin, Relo
     private static final String PRIORITY_STREAM_READER = "priority_stream_reader";
     private static final String FUTURE_COMPLETION = "future_completion";
     private static final String STREAM_READER = "stream_reader";
+    private static final String LOW_TRANSFER_QUEUE_CONSUMER = "low_transfer_queue_consumer";
+    private static final String NORMAL_TRANSFER_QUEUE_CONSUMER = "normal_transfer_queue_consumer";
 
     protected final S3Service service;
     private final S3AsyncService s3AsyncService;
@@ -91,6 +100,12 @@ public class S3RepositoryPlugin extends Plugin implements RepositoryPlugin, Relo
     private AsyncExecutorContainer urgentExecutorBuilder;
     private AsyncExecutorContainer priorityExecutorBuilder;
     private AsyncExecutorContainer normalExecutorBuilder;
+    private ExecutorService lowTransferQConsumerService;
+    private ExecutorService normalTransferQConsumerService;
+    private SizeBasedBlockingQ normalPrioritySizeBasedBlockingQ;
+    private SizeBasedBlockingQ lowPrioritySizeBasedBlockingQ;
+    private TransferSemaphoresHolder transferSemaphoresHolder;
+    private GenericStatsMetricPublisher genericStatsMetricPublisher;
 
     public S3RepositoryPlugin(final Settings settings, final Path configPath) {
         this(settings, configPath, new S3Service(configPath), new S3AsyncService(configPath));
@@ -120,9 +135,36 @@ public List<ExecutorBuilder<?>> getExecutorBuilders(Settings settings) {
                 TimeValue.timeValueMinutes(5)
             )
         );
+        executorBuilders.add(
+            new FixedExecutorBuilder(
+                settings,
+                LOW_TRANSFER_QUEUE_CONSUMER,
+                lowPriorityTransferQConsumers(settings),
+                10,
+                "thread_pool." + LOW_TRANSFER_QUEUE_CONSUMER
+            )
+        );
+        executorBuilders.add(
+            new FixedExecutorBuilder(
+                settings,
+                NORMAL_TRANSFER_QUEUE_CONSUMER,
+                normalPriorityTransferQConsumers(settings),
+                10,
+                "thread_pool." + NORMAL_TRANSFER_QUEUE_CONSUMER
+            )
+        );
         return executorBuilders;
     }
 
+    private int lowPriorityTransferQConsumers(Settings settings) {
+        double lowPriorityAllocation = ((double) (100 - S3Repository.S3_PRIORITY_PERMIT_ALLOCATION_PERCENT.get(settings))) / 100;
+        return Math.max(2, (int) (lowPriorityAllocation * S3Repository.S3_TRANSFER_QUEUE_CONSUMERS.get(settings)));
+    }
+
+    private int normalPriorityTransferQConsumers(Settings settings) {
+        return S3Repository.S3_TRANSFER_QUEUE_CONSUMERS.get(settings);
+    }
+
     static int halfNumberOfProcessors(int numberOfProcessors) {
         return (numberOfProcessors + 1) / 2;
     }
@@ -189,7 +231,67 @@ public Collection<Object> createComponents(
             threadPool.executor(STREAM_READER),
             new AsyncTransferEventLoopGroup(normalEventLoopThreads)
         );
-        return Collections.emptyList();
+
+        this.lowTransferQConsumerService = threadPool.executor(LOW_TRANSFER_QUEUE_CONSUMER);
+        this.normalTransferQConsumerService = threadPool.executor(NORMAL_TRANSFER_QUEUE_CONSUMER);
+
+        // High number of permit allocation because each op acquiring permit performs disk IO, computation and network IO.
+        int availablePermits = Math.max(allocatedProcessors(clusterService.getSettings()) * 4, 10);
+        double priorityPermitAllocation = ((double) S3Repository.S3_PRIORITY_PERMIT_ALLOCATION_PERCENT.get(clusterService.getSettings()))
+            / 100;
+        int normalPriorityPermits = (int) (priorityPermitAllocation * availablePermits);
+        int lowPriorityPermits = availablePermits - normalPriorityPermits;
+
+        int normalPriorityConsumers = normalPriorityTransferQConsumers(clusterService.getSettings());
+        int lowPriorityConsumers = lowPriorityTransferQConsumers(clusterService.getSettings());
+
+        ByteSizeValue normalPriorityQCapacity = new ByteSizeValue(normalPriorityConsumers * 10L, ByteSizeUnit.GB);
+        ByteSizeValue lowPriorityQCapacity = new ByteSizeValue(lowPriorityConsumers * 20L, ByteSizeUnit.GB);
+
+        this.genericStatsMetricPublisher = new GenericStatsMetricPublisher(
+            normalPriorityQCapacity.getBytes(),
+            normalPriorityPermits,
+            lowPriorityQCapacity.getBytes(),
+            lowPriorityPermits
+        );
+
+        this.normalPrioritySizeBasedBlockingQ = new SizeBasedBlockingQ(
+            normalPriorityQCapacity,
+            normalTransferQConsumerService,
+            normalPriorityConsumers,
+            genericStatsMetricPublisher,
+            SizeBasedBlockingQ.QueueEventType.NORMAL
+        );
+
+        LowPrioritySizeBasedBlockingQ lowPrioritySizeBasedBlockingQ = new LowPrioritySizeBasedBlockingQ(
+            lowPriorityQCapacity,
+            lowTransferQConsumerService,
+            lowPriorityConsumers,
+            genericStatsMetricPublisher
+        );
+        this.lowPrioritySizeBasedBlockingQ = lowPrioritySizeBasedBlockingQ;
+        this.transferSemaphoresHolder = new TransferSemaphoresHolder(
+            normalPriorityPermits,
+            lowPriorityPermits,
+            S3Repository.S3_PERMIT_WAIT_DURATION_MIN.get(clusterService.getSettings()),
+            TimeUnit.MINUTES,
+            genericStatsMetricPublisher
+        );
+
+        return CollectionUtils.arrayAsArrayList(this.normalPrioritySizeBasedBlockingQ, lowPrioritySizeBasedBlockingQ);
+    }
+
+    // New class because in core, components are injected via guice only by instance creation due to which
+    // same binding types fail.
+    private static final class LowPrioritySizeBasedBlockingQ extends SizeBasedBlockingQ {
+        public LowPrioritySizeBasedBlockingQ(
+            ByteSizeValue capacity,
+            ExecutorService executorService,
+            int consumers,
+            GenericStatsMetricPublisher genericStatsMetricPublisher
+        ) {
+            super(capacity, executorService, consumers, genericStatsMetricPublisher, QueueEventType.LOW);
+        }
     }
 
     // proxy method for testing
@@ -204,7 +306,8 @@ protected S3Repository createRepository(
             S3Repository.PARALLEL_MULTIPART_UPLOAD_MINIMUM_PART_SIZE_SETTING.get(clusterService.getSettings()).getBytes(),
             normalExecutorBuilder.getStreamReader(),
             priorityExecutorBuilder.getStreamReader(),
-            urgentExecutorBuilder.getStreamReader()
+            urgentExecutorBuilder.getStreamReader(),
+            transferSemaphoresHolder
         );
         return new S3Repository(
             metadata,
@@ -218,7 +321,10 @@ protected S3Repository createRepository(
             normalExecutorBuilder,
             s3AsyncService,
             S3Repository.PARALLEL_MULTIPART_UPLOAD_ENABLED_SETTING.get(clusterService.getSettings()),
-            configPath
+            configPath,
+            normalPrioritySizeBasedBlockingQ,
+            lowPrioritySizeBasedBlockingQ,
+            genericStatsMetricPublisher
         );
     }
 
@@ -263,7 +369,9 @@ public List<Setting<?>> getSettings() {
             S3Repository.PARALLEL_MULTIPART_UPLOAD_MINIMUM_PART_SIZE_SETTING,
             S3Repository.PARALLEL_MULTIPART_UPLOAD_ENABLED_SETTING,
             S3Repository.REDIRECT_LARGE_S3_UPLOAD,
-            S3Repository.UPLOAD_RETRY_ENABLED
+            S3Repository.UPLOAD_RETRY_ENABLED,
+            S3Repository.S3_PRIORITY_PERMIT_ALLOCATION_PERCENT,
+            S3Repository.PERMIT_BACKED_TRANSFER_ENABLED
         );
     }
 
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RetryingInputStream.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RetryingInputStream.java
index d7e47e0ab1bcc..2eb63178b19e0 100644
--- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RetryingInputStream.java
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RetryingInputStream.java
@@ -48,6 +48,7 @@
 import java.nio.file.NoSuchFileException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
@@ -77,6 +78,7 @@ class S3RetryingInputStream extends InputStream {
     private long currentOffset;
     private boolean closed;
     private boolean eof;
+    private Map<String, String> metadata;
 
     S3RetryingInputStream(S3BlobStore blobStore, String blobKey) throws IOException {
         this(blobStore, blobKey, 0, Long.MAX_VALUE - 1);
@@ -122,6 +124,7 @@ private void openStream() throws IOException {
                 getObjectResponseInputStream.response().contentLength()
             );
             this.currentStream = getObjectResponseInputStream;
+            this.metadata = getObjectResponseInputStream.response().metadata();
             this.isStreamAborted.set(false);
         } catch (final SdkException e) {
             if (e instanceof S3Exception) {
@@ -265,4 +268,8 @@ boolean isEof() {
     boolean isAborted() {
         return isStreamAborted.get();
     }
+
+    Map<String, String> getMetadata() {
+        return this.metadata;
+    }
 }
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3TransferRejectedException.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3TransferRejectedException.java
new file mode 100644
index 0000000000000..c9fa93ea0f5c3
--- /dev/null
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3TransferRejectedException.java
@@ -0,0 +1,20 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.repositories.s3;
+
+import org.opensearch.OpenSearchException;
+
+/**
+ * Thrown when transfer event is rejected due to breach in event queue size.
+ */
+public class S3TransferRejectedException extends OpenSearchException {
+    public S3TransferRejectedException(String msg) {
+        super(msg);
+    }
+}
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncPartsHandler.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncPartsHandler.java
index b4c4ed0ecaa75..4c95a0ffc5ec3 100644
--- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncPartsHandler.java
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncPartsHandler.java
@@ -23,6 +23,7 @@
 import org.opensearch.common.StreamContext;
 import org.opensearch.common.blobstore.stream.write.WritePriority;
 import org.opensearch.common.io.InputStreamContainer;
+import org.opensearch.repositories.s3.S3TransferRejectedException;
 import org.opensearch.repositories.s3.SocketAccess;
 import org.opensearch.repositories.s3.StatsMetricPublisher;
 import org.opensearch.repositories.s3.io.CheckedContainer;
@@ -34,6 +35,8 @@
 import java.util.List;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReferenceArray;
 
 /**
@@ -41,7 +44,7 @@
  */
 public class AsyncPartsHandler {
 
-    private static Logger log = LogManager.getLogger(AsyncPartsHandler.class);
+    private static final Logger log = LogManager.getLogger(AsyncPartsHandler.class);
 
     /**
      * Uploads parts of the upload multipart request*
@@ -55,9 +58,10 @@ public class AsyncPartsHandler {
      * @param completedParts Reference of completed parts
      * @param inputStreamContainers Checksum containers
      * @param statsMetricPublisher sdk metric publisher
+     * @param maxRetryablePartSize Max content size which can be used for retries in buffered streams.
      * @return list of completable futures
-     * @throws IOException thrown in case of an IO error
      */
+    @SuppressWarnings({ "rawtypes", "unchecked" })
     public static List<CompletableFuture<CompletedPart>> uploadParts(
         S3AsyncClient s3AsyncClient,
         ExecutorService executorService,
@@ -69,35 +73,52 @@ public static List<CompletableFuture<CompletedPart>> uploadParts(
         AtomicReferenceArray<CompletedPart> completedParts,
         AtomicReferenceArray<CheckedContainer> inputStreamContainers,
         StatsMetricPublisher statsMetricPublisher,
-        boolean uploadRetryEnabled
-    ) throws IOException {
+        boolean uploadRetryEnabled,
+        TransferSemaphoresHolder transferSemaphoresHolder,
+        long maxRetryablePartSize
+    ) throws InterruptedException {
         List<CompletableFuture<CompletedPart>> futures = new ArrayList<>();
+        TransferSemaphoresHolder.RequestContext requestContext = transferSemaphoresHolder.createRequestContext();
         for (int partIdx = 0; partIdx < streamContext.getNumberOfParts(); partIdx++) {
-            InputStreamContainer inputStreamContainer = streamContext.provideStream(partIdx);
-            inputStreamContainers.set(partIdx, new CheckedContainer(inputStreamContainer.getContentLength()));
-            UploadPartRequest.Builder uploadPartRequestBuilder = UploadPartRequest.builder()
-                .bucket(uploadRequest.getBucket())
-                .partNumber(partIdx + 1)
-                .key(uploadRequest.getKey())
-                .uploadId(uploadId)
-                .overrideConfiguration(o -> o.addMetricPublisher(statsMetricPublisher.multipartUploadMetricCollector))
-                .contentLength(inputStreamContainer.getContentLength());
-            if (uploadRequest.doRemoteDataIntegrityCheck()) {
-                uploadPartRequestBuilder.checksumAlgorithm(ChecksumAlgorithm.CRC32);
-            }
-            uploadPart(
-                s3AsyncClient,
-                executorService,
-                priorityExecutorService,
-                urgentExecutorService,
-                completedParts,
-                inputStreamContainers,
-                futures,
-                uploadPartRequestBuilder.build(),
-                inputStreamContainer,
-                uploadRequest,
-                uploadRetryEnabled
+            Semaphore semaphore = maybeAcquireSemaphore(
+                transferSemaphoresHolder,
+                requestContext,
+                uploadRequest.getWritePriority(),
+                uploadRequest.getKey()
             );
+            try {
+                InputStreamContainer inputStreamContainer = streamContext.provideStream(partIdx);
+                inputStreamContainers.set(partIdx, new CheckedContainer(inputStreamContainer.getContentLength()));
+                UploadPartRequest.Builder uploadPartRequestBuilder = UploadPartRequest.builder()
+                    .bucket(uploadRequest.getBucket())
+                    .partNumber(partIdx + 1)
+                    .key(uploadRequest.getKey())
+                    .uploadId(uploadId)
+                    .overrideConfiguration(o -> o.addMetricPublisher(statsMetricPublisher.multipartUploadMetricCollector))
+                    .contentLength(inputStreamContainer.getContentLength());
+                if (uploadRequest.doRemoteDataIntegrityCheck()) {
+                    uploadPartRequestBuilder.checksumAlgorithm(ChecksumAlgorithm.CRC32);
+                }
+                uploadPart(
+                    s3AsyncClient,
+                    executorService,
+                    priorityExecutorService,
+                    urgentExecutorService,
+                    completedParts,
+                    inputStreamContainers,
+                    futures,
+                    uploadPartRequestBuilder.build(),
+                    inputStreamContainer,
+                    uploadRequest,
+                    uploadRetryEnabled,
+                    maxRetryablePartSize,
+                    semaphore
+                );
+            } catch (Exception ex) {
+                if (semaphore != null) {
+                    semaphore.release();
+                }
+            }
         }
 
         return futures;
@@ -137,14 +158,54 @@ public static InputStream maybeRetryInputStream(
         InputStream inputStream,
         WritePriority writePriority,
         boolean uploadRetryEnabled,
-        long contentLength
+        long contentLength,
+        long maxRetryablePartSize
     ) {
-        if (uploadRetryEnabled == true && (writePriority == WritePriority.HIGH || writePriority == WritePriority.URGENT)) {
-            return new BufferedInputStream(inputStream, (int) (contentLength + 1));
+        // Since we are backing uploads with limited permits, it is ok to use buffered stream. Maximum in-memory buffer
+        // would be (max permits * maxRetryablePartSize) excluding urgent
+        if (uploadRetryEnabled == true
+            && (contentLength <= maxRetryablePartSize || writePriority == WritePriority.HIGH || writePriority == WritePriority.URGENT)) {
+            return new UploadTrackedBufferedInputStream(inputStream, (int) (contentLength + 1));
         }
         return inputStream;
     }
 
+    public static Semaphore maybeAcquireSemaphore(
+        TransferSemaphoresHolder transferSemaphoresHolder,
+        TransferSemaphoresHolder.RequestContext requestContext,
+        WritePriority writePriority,
+        String file
+    ) throws InterruptedException {
+        final TransferSemaphoresHolder.TypeSemaphore semaphore;
+        if (writePriority != WritePriority.HIGH && writePriority != WritePriority.URGENT) {
+            semaphore = transferSemaphoresHolder.acquirePermit(writePriority, requestContext);
+            if (semaphore == null) {
+                throw new S3TransferRejectedException("Permit not available for transfer of file " + file);
+            }
+        } else {
+            semaphore = null;
+        }
+
+        return semaphore;
+    }
+
+    /**
+     * Overridden stream to identify upload streams among all buffered stream instances for triaging.
+     */
+    static class UploadTrackedBufferedInputStream extends BufferedInputStream {
+        AtomicBoolean closed = new AtomicBoolean();
+
+        public UploadTrackedBufferedInputStream(InputStream in, int length) {
+            super(in, length);
+        }
+
+        @Override
+        public void close() throws IOException {
+            super.close();
+            closed.set(true);
+        }
+    }
+
     private static void uploadPart(
         S3AsyncClient s3AsyncClient,
         ExecutorService executorService,
@@ -156,8 +217,11 @@ private static void uploadPart(
         UploadPartRequest uploadPartRequest,
         InputStreamContainer inputStreamContainer,
         UploadRequest uploadRequest,
-        boolean uploadRetryEnabled
+        boolean uploadRetryEnabled,
+        long maxRetryablePartSize,
+        Semaphore semaphore
     ) {
+
         Integer partNumber = uploadPartRequest.partNumber();
 
         ExecutorService streamReadExecutor;
@@ -173,7 +237,8 @@ private static void uploadPart(
             inputStreamContainer.getInputStream(),
             uploadRequest.getWritePriority(),
             uploadRetryEnabled,
-            uploadPartRequest.contentLength()
+            uploadPartRequest.contentLength(),
+            maxRetryablePartSize
         );
         CompletableFuture<UploadPartResponse> uploadPartResponseFuture = SocketAccess.doPrivileged(
             () -> s3AsyncClient.uploadPart(
@@ -183,6 +248,10 @@ private static void uploadPart(
         );
 
         CompletableFuture<CompletedPart> convertFuture = uploadPartResponseFuture.whenComplete((resp, throwable) -> {
+            if (semaphore != null) {
+                semaphore.release();
+            }
+
             try {
                 inputStream.close();
             } catch (IOException ex) {
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncTransferManager.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncTransferManager.java
index 2259780c95276..0f9bf3be77d73 100644
--- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncTransferManager.java
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/AsyncTransferManager.java
@@ -21,7 +21,9 @@
 import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse;
 import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
 import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.awssdk.services.s3.model.PutObjectResponse;
 import software.amazon.awssdk.services.s3.model.S3Exception;
+import software.amazon.awssdk.utils.CollectionUtils;
 import software.amazon.awssdk.utils.CompletableFutureUtils;
 
 import org.apache.logging.log4j.LogManager;
@@ -47,6 +49,7 @@
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionException;
 import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Semaphore;
 import java.util.concurrent.atomic.AtomicReferenceArray;
 import java.util.function.BiFunction;
 import java.util.function.Supplier;
@@ -63,6 +66,10 @@ public final class AsyncTransferManager {
     private final ExecutorService priorityExecutorService;
     private final ExecutorService urgentExecutorService;
     private final long minimumPartSize;
+    private final long maxRetryablePartSize;
+
+    @SuppressWarnings("rawtypes")
+    private final TransferSemaphoresHolder transferSemaphoresHolder;
 
     /**
      * The max number of parts on S3 side is 10,000
@@ -73,19 +80,22 @@ public final class AsyncTransferManager {
      * Construct a new object of AsyncTransferManager
      *
      * @param minimumPartSize         The minimum part size for parallel multipart uploads
-     * @param executorService         The stream reader {@link ExecutorService} for normal priority uploads
-     * @param priorityExecutorService The stream read {@link ExecutorService} for high priority uploads
      */
+    @SuppressWarnings("rawtypes")
     public AsyncTransferManager(
         long minimumPartSize,
         ExecutorService executorService,
         ExecutorService priorityExecutorService,
-        ExecutorService urgentExecutorService
+        ExecutorService urgentExecutorService,
+        TransferSemaphoresHolder transferSemaphoresHolder
     ) {
         this.executorService = executorService;
         this.priorityExecutorService = priorityExecutorService;
         this.minimumPartSize = minimumPartSize;
+        // 10% buffer to allow additional metadata size in content such as encryption.
+        this.maxRetryablePartSize = (long) (minimumPartSize + 0.1 * minimumPartSize);
         this.urgentExecutorService = urgentExecutorService;
+        this.transferSemaphoresHolder = transferSemaphoresHolder;
     }
 
     /**
@@ -107,7 +117,21 @@ public CompletableFuture<Void> uploadObject(
         try {
             if (streamContext.getNumberOfParts() == 1) {
                 log.debug(() -> "Starting the upload as a single upload part request");
-                uploadInOneChunk(s3AsyncClient, uploadRequest, streamContext.provideStream(0), returnFuture, statsMetricPublisher);
+                TransferSemaphoresHolder.RequestContext requestContext = transferSemaphoresHolder.createRequestContext();
+                Semaphore semaphore = AsyncPartsHandler.maybeAcquireSemaphore(
+                    transferSemaphoresHolder,
+                    requestContext,
+                    uploadRequest.getWritePriority(),
+                    uploadRequest.getKey()
+                );
+                try {
+                    uploadInOneChunk(s3AsyncClient, uploadRequest, streamContext, returnFuture, statsMetricPublisher, semaphore);
+                } catch (Exception ex) {
+                    if (semaphore != null) {
+                        semaphore.release();
+                    }
+                    throw ex;
+                }
             } else {
                 log.debug(() -> "Starting the upload as multipart upload request");
                 uploadInParts(s3AsyncClient, uploadRequest, streamContext, returnFuture, statsMetricPublisher);
@@ -131,6 +155,10 @@ private void uploadInParts(
             .bucket(uploadRequest.getBucket())
             .key(uploadRequest.getKey())
             .overrideConfiguration(o -> o.addMetricPublisher(statsMetricPublisher.multipartUploadMetricCollector));
+
+        if (CollectionUtils.isNotEmpty(uploadRequest.getMetadata())) {
+            createMultipartUploadRequestBuilder.metadata(uploadRequest.getMetadata());
+        }
         if (uploadRequest.doRemoteDataIntegrityCheck()) {
             createMultipartUploadRequestBuilder.checksumAlgorithm(ChecksumAlgorithm.CRC32);
         }
@@ -141,21 +169,19 @@ private void uploadInParts(
         // Ensure cancellations are forwarded to the createMultipartUploadFuture future
         CompletableFutureUtils.forwardExceptionTo(returnFuture, createMultipartUploadFuture);
 
-        createMultipartUploadFuture.whenComplete((createMultipartUploadResponse, throwable) -> {
-            if (throwable != null) {
-                handleException(returnFuture, () -> "Failed to initiate multipart upload", throwable);
-            } else {
-                log.debug(() -> "Initiated new multipart upload, uploadId: " + createMultipartUploadResponse.uploadId());
-                doUploadInParts(
-                    s3AsyncClient,
-                    uploadRequest,
-                    streamContext,
-                    returnFuture,
-                    createMultipartUploadResponse.uploadId(),
-                    statsMetricPublisher
-                );
-            }
-        });
+        String uploadId;
+        try {
+            // Block main thread here so that upload of parts doesn't get executed in future completion thread.
+            // We should never execute latent operation like acquisition of permit in future completion pool.
+            CreateMultipartUploadResponse createMultipartUploadResponse = createMultipartUploadFuture.get();
+            uploadId = createMultipartUploadResponse.uploadId();
+            log.debug(() -> "Initiated new multipart upload, uploadId: " + createMultipartUploadResponse.uploadId());
+        } catch (Exception ex) {
+            handleException(returnFuture, () -> "Failed to initiate multipart upload", ex);
+            return;
+        }
+
+        doUploadInParts(s3AsyncClient, uploadRequest, streamContext, returnFuture, uploadId, statsMetricPublisher);
     }
 
     private void doUploadInParts(
@@ -184,7 +210,9 @@ private void doUploadInParts(
                 completedParts,
                 inputStreamContainers,
                 statsMetricPublisher,
-                uploadRequest.isUploadRetryEnabled()
+                uploadRequest.isUploadRetryEnabled(),
+                transferSemaphoresHolder,
+                maxRetryablePartSize
             );
         } catch (Exception ex) {
             try {
@@ -315,22 +343,29 @@ public long calculateOptimalPartSize(long contentLengthOfSource, WritePriority w
         return (long) Math.max(optimalPartSize, minimumPartSize);
     }
 
+    @SuppressWarnings("unchecked")
     private void uploadInOneChunk(
         S3AsyncClient s3AsyncClient,
         UploadRequest uploadRequest,
-        InputStreamContainer inputStreamContainer,
+        StreamContext streamContext,
         CompletableFuture<Void> returnFuture,
-        StatsMetricPublisher statsMetricPublisher
+        StatsMetricPublisher statsMetricPublisher,
+        Semaphore semaphore
     ) {
         PutObjectRequest.Builder putObjectRequestBuilder = PutObjectRequest.builder()
             .bucket(uploadRequest.getBucket())
             .key(uploadRequest.getKey())
             .contentLength(uploadRequest.getContentLength())
             .overrideConfiguration(o -> o.addMetricPublisher(statsMetricPublisher.putObjectMetricPublisher));
+
+        if (CollectionUtils.isNotEmpty(uploadRequest.getMetadata())) {
+            putObjectRequestBuilder.metadata(uploadRequest.getMetadata());
+        }
         if (uploadRequest.doRemoteDataIntegrityCheck()) {
             putObjectRequestBuilder.checksumAlgorithm(ChecksumAlgorithm.CRC32);
             putObjectRequestBuilder.checksumCRC32(base64StringFromLong(uploadRequest.getExpectedChecksum()));
         }
+        PutObjectRequest putObjectRequest = putObjectRequestBuilder.build();
         ExecutorService streamReadExecutor;
         if (uploadRequest.getWritePriority() == WritePriority.URGENT) {
             streamReadExecutor = urgentExecutorService;
@@ -340,25 +375,33 @@ private void uploadInOneChunk(
             streamReadExecutor = executorService;
         }
 
-        InputStream inputStream = AsyncPartsHandler.maybeRetryInputStream(
-            inputStreamContainer.getInputStream(),
-            uploadRequest.getWritePriority(),
-            uploadRequest.isUploadRetryEnabled(),
-            uploadRequest.getContentLength()
-        );
-        CompletableFuture<Void> putObjectFuture = SocketAccess.doPrivileged(
-            () -> s3AsyncClient.putObject(
-                putObjectRequestBuilder.build(),
-                AsyncRequestBody.fromInputStream(inputStream, inputStreamContainer.getContentLength(), streamReadExecutor)
-            ).handle((resp, throwable) -> {
-                try {
-                    inputStream.close();
-                } catch (IOException e) {
-                    log.error(
-                        () -> new ParameterizedMessage("Failed to close stream while uploading single file {}.", uploadRequest.getKey()),
-                        e
-                    );
-                }
+        CompletableFuture<Void> putObjectFuture = SocketAccess.doPrivileged(() -> {
+            InputStream inputStream = null;
+            CompletableFuture<PutObjectResponse> putObjectRespFuture;
+            try {
+                InputStreamContainer inputStreamContainer = streamContext.provideStream(0);
+                inputStream = AsyncPartsHandler.maybeRetryInputStream(
+                    inputStreamContainer.getInputStream(),
+                    uploadRequest.getWritePriority(),
+                    uploadRequest.isUploadRetryEnabled(),
+                    uploadRequest.getContentLength(),
+                    maxRetryablePartSize
+                );
+                AsyncRequestBody asyncRequestBody = AsyncRequestBody.fromInputStream(
+                    inputStream,
+                    inputStreamContainer.getContentLength(),
+                    streamReadExecutor
+                );
+                putObjectRespFuture = s3AsyncClient.putObject(putObjectRequest, asyncRequestBody);
+            } catch (Exception e) {
+                releaseResourcesSafely(semaphore, inputStream, uploadRequest.getKey());
+                return CompletableFuture.failedFuture(e);
+            }
+
+            InputStream finalInputStream = inputStream;
+            return putObjectRespFuture.handle((resp, throwable) -> {
+                releaseResourcesSafely(semaphore, finalInputStream, uploadRequest.getKey());
+
                 if (throwable != null) {
                     Throwable unwrappedThrowable = ExceptionsHelper.unwrap(throwable, S3Exception.class);
                     if (unwrappedThrowable != null) {
@@ -386,13 +429,27 @@ private void uploadInOneChunk(
                 }
 
                 return null;
-            })
-        );
+            });
+        });
 
         CompletableFutureUtils.forwardExceptionTo(returnFuture, putObjectFuture);
         CompletableFutureUtils.forwardResultTo(putObjectFuture, returnFuture);
     }
 
+    private void releaseResourcesSafely(Semaphore semaphore, InputStream inputStream, String file) {
+        if (semaphore != null) {
+            semaphore.release();
+        }
+
+        if (inputStream != null) {
+            try {
+                inputStream.close();
+            } catch (IOException e) {
+                log.error(() -> new ParameterizedMessage("Failed to close stream while uploading single file {}.", file), e);
+            }
+        }
+    }
+
     private void deleteUploadedObject(S3AsyncClient s3AsyncClient, UploadRequest uploadRequest) {
         DeleteObjectRequest deleteObjectRequest = DeleteObjectRequest.builder()
             .bucket(uploadRequest.getBucket())
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/SizeBasedBlockingQ.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/SizeBasedBlockingQ.java
new file mode 100644
index 0000000000000..170c80f5d4db6
--- /dev/null
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/SizeBasedBlockingQ.java
@@ -0,0 +1,230 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.repositories.s3.async;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.lucene.store.AlreadyClosedException;
+import org.opensearch.common.lifecycle.AbstractLifecycleComponent;
+import org.opensearch.core.common.unit.ByteSizeValue;
+import org.opensearch.repositories.s3.GenericStatsMetricPublisher;
+import org.opensearch.repositories.s3.S3TransferRejectedException;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+/**
+ * Queue implementation to accept events based on their storage attribute. If size of queue is breached, then transfer
+ * event is rejected.
+ */
+public class SizeBasedBlockingQ extends AbstractLifecycleComponent {
+    private static final Logger log = LogManager.getLogger(SizeBasedBlockingQ.class);
+
+    protected final LinkedBlockingQueue<Item> queue;
+    protected final Lock lock;
+    protected final Condition notEmpty;
+
+    protected final AtomicLong currentSize;
+    protected final ByteSizeValue capacity;
+    protected final AtomicBoolean closed;
+    protected final ExecutorService executorService;
+    protected final int consumers;
+    private final GenericStatsMetricPublisher genericStatsMetricPublisher;
+    private final QueueEventType queueEventType;
+
+    /**
+     * Constructor to create sized based blocking queue.
+     */
+    public SizeBasedBlockingQ(
+        ByteSizeValue capacity,
+        ExecutorService executorService,
+        int consumers,
+        GenericStatsMetricPublisher genericStatsMetricPublisher,
+        QueueEventType queueEventType
+    ) {
+        this.queue = new LinkedBlockingQueue<>();
+        this.lock = new ReentrantLock();
+        this.notEmpty = lock.newCondition();
+        this.currentSize = new AtomicLong();
+        this.capacity = capacity;
+        this.closed = new AtomicBoolean();
+        this.executorService = executorService;
+        this.consumers = consumers;
+        this.genericStatsMetricPublisher = genericStatsMetricPublisher;
+        this.queueEventType = queueEventType;
+    }
+
+    public enum QueueEventType {
+        NORMAL,
+        LOW;
+    }
+
+    @Override
+    protected void doStart() {
+        for (int worker = 0; worker < consumers; worker++) {
+            Thread consumer = new Consumer(queue, currentSize, lock, notEmpty, closed, genericStatsMetricPublisher, queueEventType);
+            executorService.submit(consumer);
+        }
+    }
+
+    /**
+     * Add an item to the queue
+     */
+    public void produce(Item item) throws InterruptedException {
+        if (item == null || item.size <= 0) {
+            throw new IllegalStateException("Invalid item input to produce.");
+        }
+        log.debug(() -> "Transfer queue event received of size: " + item.size + ". Current queue utilisation: " + currentSize.get());
+
+        if (currentSize.get() + item.size >= capacity.getBytes()) {
+            throw new S3TransferRejectedException("S3 Transfer queue capacity reached");
+        }
+
+        final Lock lock = this.lock;
+        final AtomicLong currentSize = this.currentSize;
+        lock.lock();
+        try {
+            if (currentSize.get() + item.size >= capacity.getBytes()) {
+                throw new S3TransferRejectedException("S3 Transfer queue capacity reached");
+            }
+            if (closed.get()) {
+                throw new AlreadyClosedException("Transfer queue is already closed.");
+            }
+            queue.put(item);
+            currentSize.addAndGet(item.size);
+            notEmpty.signalAll();
+            updateStats(item.size, queueEventType, genericStatsMetricPublisher);
+        } finally {
+            lock.unlock();
+        }
+
+    }
+
+    private static void updateStats(long itemSize, QueueEventType queueEventType, GenericStatsMetricPublisher genericStatsMetricPublisher) {
+        if (queueEventType == QueueEventType.NORMAL) {
+            genericStatsMetricPublisher.updateNormalPriorityQSize(itemSize);
+        } else if (queueEventType == QueueEventType.LOW) {
+            genericStatsMetricPublisher.updateLowPriorityQSize(itemSize);
+        }
+    }
+
+    public int getSize() {
+        return queue.size();
+    }
+
+    public boolean isMaxCapacityBelowContentLength(long contentLength) {
+        return contentLength < capacity.getBytes();
+    }
+
+    protected static class Consumer extends Thread {
+        private final LinkedBlockingQueue<Item> queue;
+        private final Lock lock;
+        private final Condition notEmpty;
+        private final AtomicLong currentSize;
+        private final AtomicBoolean closed;
+        private final GenericStatsMetricPublisher genericStatsMetricPublisher;
+        private final QueueEventType queueEventType;
+
+        public Consumer(
+            LinkedBlockingQueue<Item> queue,
+            AtomicLong currentSize,
+            Lock lock,
+            Condition notEmpty,
+            AtomicBoolean closed,
+            GenericStatsMetricPublisher genericStatsMetricPublisher,
+            QueueEventType queueEventType
+        ) {
+            this.queue = queue;
+            this.lock = lock;
+            this.notEmpty = notEmpty;
+            this.currentSize = currentSize;
+            this.closed = closed;
+            this.genericStatsMetricPublisher = genericStatsMetricPublisher;
+            this.queueEventType = queueEventType;
+        }
+
+        @Override
+        public void run() {
+            while (true) {
+                try {
+                    consume();
+                } catch (AlreadyClosedException ex) {
+                    return;
+                } catch (Exception ex) {
+                    log.error("Failed to consume transfer event", ex);
+                }
+            }
+        }
+
+        private void consume() throws InterruptedException {
+            final Lock lock = this.lock;
+            final AtomicLong currentSize = this.currentSize;
+            lock.lock();
+            Item item;
+            try {
+                if (closed.get()) {
+                    throw new AlreadyClosedException("transfer queue closed");
+                }
+                while (currentSize.get() == 0) {
+                    notEmpty.await();
+                    if (closed.get()) {
+                        throw new AlreadyClosedException("transfer queue closed");
+                    }
+                }
+
+                item = queue.take();
+                currentSize.addAndGet(-item.size);
+                updateStats(-item.size, queueEventType, genericStatsMetricPublisher);
+            } finally {
+                lock.unlock();
+            }
+
+            try {
+                item.consumable.run();
+            } catch (Exception ex) {
+                log.error("Exception on executing item consumable", ex);
+            }
+        }
+
+    }
+
+    public static class Item {
+        private final long size;
+        private final Runnable consumable;
+
+        public Item(long size, Runnable consumable) {
+            this.size = size;
+            this.consumable = consumable;
+        }
+    }
+
+    @Override
+    protected void doStop() {
+        doClose();
+    }
+
+    @Override
+    protected void doClose() {
+        lock.lock();
+        try {
+            if (closed.get() == true) {
+                return;
+            }
+            closed.set(true);
+            notEmpty.signalAll();
+        } finally {
+            lock.unlock();
+        }
+    }
+}
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/TransferSemaphoresHolder.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/TransferSemaphoresHolder.java
new file mode 100644
index 0000000000000..7dccedb8d5278
--- /dev/null
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/TransferSemaphoresHolder.java
@@ -0,0 +1,186 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.repositories.s3.async;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.opensearch.common.blobstore.stream.write.WritePriority;
+import org.opensearch.repositories.s3.GenericStatsMetricPublisher;
+
+import java.util.Objects;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
+
+/**
+ * Transfer semaphore holder for controlled transfer of data to remote.
+ */
+public class TransferSemaphoresHolder {
+    private static final Logger log = LogManager.getLogger(TransferSemaphoresHolder.class);
+    // For tests
+    protected TypeSemaphore lowPrioritySemaphore;
+    protected TypeSemaphore normalPrioritySemaphore;
+    private final int normalPriorityPermits;
+    private final int lowPriorityPermits;
+    private final int acquireWaitDuration;
+    private final TimeUnit acquireWaitDurationUnit;
+
+    /**
+     * Constructor to create semaphores holder.
+     */
+    public TransferSemaphoresHolder(
+        int normalPriorityPermits,
+        int lowPriorityPermits,
+        int acquireWaitDuration,
+        TimeUnit timeUnit,
+        GenericStatsMetricPublisher genericStatsPublisher
+    ) {
+
+        this.normalPriorityPermits = normalPriorityPermits;
+        this.lowPriorityPermits = lowPriorityPermits;
+        this.normalPrioritySemaphore = new TypeSemaphore(
+            normalPriorityPermits,
+            TypeSemaphore.PermitType.NORMAL,
+            genericStatsPublisher::updateNormalPermits
+        );
+        this.lowPrioritySemaphore = new TypeSemaphore(
+            lowPriorityPermits,
+            TypeSemaphore.PermitType.LOW,
+            genericStatsPublisher::updateLowPermits
+        );
+        this.acquireWaitDuration = acquireWaitDuration;
+        this.acquireWaitDurationUnit = timeUnit;
+    }
+
+    /**
+     * Overridden semaphore to identify transfer semaphores among all other semaphores for triaging.
+     */
+    public static class TypeSemaphore extends Semaphore {
+        private final PermitType permitType;
+        private final Consumer<Boolean> permitChangeConsumer;
+
+        public enum PermitType {
+            NORMAL,
+            LOW;
+        }
+
+        public TypeSemaphore(int permits, PermitType permitType, Consumer<Boolean> permitChangeConsumer) {
+            super(permits);
+            this.permitType = permitType;
+            this.permitChangeConsumer = permitChangeConsumer;
+        }
+
+        public PermitType getType() {
+            return permitType;
+        }
+
+        @Override
+        public boolean tryAcquire() {
+            boolean acquired = super.tryAcquire();
+            if (acquired) {
+                permitChangeConsumer.accept(true);
+            }
+            return acquired;
+        }
+
+        @Override
+        public boolean tryAcquire(long timeout, TimeUnit unit) throws InterruptedException {
+            boolean acquired = super.tryAcquire(timeout, unit);
+            if (acquired) {
+                permitChangeConsumer.accept(true);
+            }
+            return acquired;
+        }
+
+        @Override
+        public void release() {
+            super.release();
+            permitChangeConsumer.accept(false);
+        }
+    }
+
+    /**
+     * For multiple part requests of a single file, request context object will be set with the decision if low
+     * priority permits can also be utilized in high priority transfers of parts of the file. If high priority get fully
+     * consumed then low priority permits will be acquired for transfer.
+     *
+     * If a low priority transfer request comes in and a high priority transfer is in progress then till current
+     * high priority transfer finishes, low priority transfer may have to compete. This is an acceptable side effect
+     * because low priority transfers are generally heavy and it is ok to have slow progress in the beginning.
+     *
+     */
+    public static class RequestContext {
+
+        private final boolean lowPriorityPermitsConsumable;
+
+        private RequestContext(boolean lowPriorityPermitsConsumable) {
+            this.lowPriorityPermitsConsumable = lowPriorityPermitsConsumable;
+        }
+
+    }
+
+    public RequestContext createRequestContext() {
+        return new RequestContext(this.lowPrioritySemaphore.availablePermits() == lowPriorityPermits);
+    }
+
+    /**
+     * Acquire permit based on the availability and based on the transfer priority.
+     * A high priority event can acquire a low priority semaphore if all low permits are available.
+     * A low priority event can acquire a high priority semaphore if at least 40% of high permits are available. We
+     * reserve this bandwidth to ensure that high priority events never wait for permits in case of ongoing low priority
+     * transfers.
+     */
+    public TypeSemaphore acquirePermit(WritePriority writePriority, RequestContext requestContext) throws InterruptedException {
+        log.debug(
+            () -> "Acquire permit request for transfer type: "
+                + writePriority
+                + ". Available high priority permits: "
+                + normalPrioritySemaphore.availablePermits()
+                + " and low priority permits: "
+                + lowPrioritySemaphore.availablePermits()
+        );
+        // Try acquiring low priority permit or high priority permit immediately if available.
+        // Otherwise, we wait for low priority permit.
+        if (Objects.requireNonNull(writePriority) == WritePriority.LOW) {
+            if (lowPrioritySemaphore.tryAcquire()) {
+                return lowPrioritySemaphore;
+            } else if (normalPrioritySemaphore.availablePermits() > 0.4 * normalPriorityPermits && normalPrioritySemaphore.tryAcquire()) {
+                return normalPrioritySemaphore;
+            } else if (lowPrioritySemaphore.tryAcquire(acquireWaitDuration, acquireWaitDurationUnit)) {
+                return lowPrioritySemaphore;
+            }
+            return null;
+        }
+
+        // Try acquiring high priority permit or low priority permit immediately if available.
+        // Otherwise, we wait for high priority permit.
+        if (normalPrioritySemaphore.tryAcquire()) {
+            return normalPrioritySemaphore;
+        } else if (requestContext.lowPriorityPermitsConsumable && lowPrioritySemaphore.tryAcquire()) {
+            return lowPrioritySemaphore;
+        } else if (normalPrioritySemaphore.tryAcquire(acquireWaitDuration, acquireWaitDurationUnit)) {
+            return normalPrioritySemaphore;
+        }
+        return null;
+    }
+
+    /**
+     * Used in tests.
+     */
+    public int getNormalPriorityPermits() {
+        return normalPriorityPermits;
+    }
+
+    /**
+     * Used in tests.
+     */
+    public int getLowPriorityPermits() {
+        return lowPriorityPermits;
+    }
+}
diff --git a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/UploadRequest.java b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/UploadRequest.java
index a5304dc4a97d6..79b58ff215c54 100644
--- a/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/UploadRequest.java
+++ b/plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/async/UploadRequest.java
@@ -9,9 +9,11 @@
 package org.opensearch.repositories.s3.async;
 
 import org.opensearch.common.CheckedConsumer;
+import org.opensearch.common.Nullable;
 import org.opensearch.common.blobstore.stream.write.WritePriority;
 
 import java.io.IOException;
+import java.util.Map;
 
 /**
  * A model encapsulating all details for an upload to S3
@@ -24,8 +26,8 @@ public class UploadRequest {
     private final CheckedConsumer<Boolean, IOException> uploadFinalizer;
     private final boolean doRemoteDataIntegrityCheck;
     private final Long expectedChecksum;
-
-    private boolean uploadRetryEnabled;
+    private final Map<String, String> metadata;
+    private final boolean uploadRetryEnabled;
 
     /**
      * Construct a new UploadRequest object
@@ -37,6 +39,7 @@ public class UploadRequest {
      * @param uploadFinalizer            An upload finalizer to call once all parts are uploaded
      * @param doRemoteDataIntegrityCheck A boolean to inform vendor plugins whether remote data integrity checks need to be done
      * @param expectedChecksum           Checksum of the file being uploaded for remote data integrity check
+     * @param metadata                   Metadata of the file being uploaded
      */
     public UploadRequest(
         String bucket,
@@ -46,7 +49,8 @@ public UploadRequest(
         CheckedConsumer<Boolean, IOException> uploadFinalizer,
         boolean doRemoteDataIntegrityCheck,
         Long expectedChecksum,
-        boolean uploadRetryEnabled
+        boolean uploadRetryEnabled,
+        @Nullable Map<String, String> metadata
     ) {
         this.bucket = bucket;
         this.key = key;
@@ -56,6 +60,7 @@ public UploadRequest(
         this.doRemoteDataIntegrityCheck = doRemoteDataIntegrityCheck;
         this.expectedChecksum = expectedChecksum;
         this.uploadRetryEnabled = uploadRetryEnabled;
+        this.metadata = metadata;
     }
 
     public String getBucket() {
@@ -89,4 +94,11 @@ public Long getExpectedChecksum() {
     public boolean isUploadRetryEnabled() {
         return uploadRetryEnabled;
     }
+
+    /**
+     * @return metadata of the blob to be uploaded
+     */
+    public Map<String, String> getMetadata() {
+        return metadata;
+    }
 }
diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/RepositoryCredentialsTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/RepositoryCredentialsTests.java
index f84d953baae8e..573a4f3f51a41 100644
--- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/RepositoryCredentialsTests.java
+++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/RepositoryCredentialsTests.java
@@ -303,7 +303,22 @@ protected S3Repository createRepository(
             ClusterService clusterService,
             RecoverySettings recoverySettings
         ) {
-            return new S3Repository(metadata, registry, service, clusterService, recoverySettings, null, null, null, null, null, false) {
+            return new S3Repository(
+                metadata,
+                registry,
+                service,
+                clusterService,
+                recoverySettings,
+                null,
+                null,
+                null,
+                null,
+                null,
+                false,
+                null,
+                null,
+                null
+            ) {
                 @Override
                 protected void assertSnapshotOrGenericThread() {
                     // eliminate thread name check as we create repo manually on test/main threads
diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerMockClientTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerMockClientTests.java
index 8c7e196d7c812..9b413ac81d766 100644
--- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerMockClientTests.java
+++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerMockClientTests.java
@@ -30,7 +30,7 @@
 
 import org.apache.lucene.store.IndexInput;
 import org.opensearch.cluster.metadata.RepositoryMetadata;
-import org.opensearch.common.CheckedTriFunction;
+import org.opensearch.common.CheckedConsumer;
 import org.opensearch.common.StreamContext;
 import org.opensearch.common.blobstore.BlobPath;
 import org.opensearch.common.blobstore.stream.write.StreamContextSupplier;
@@ -47,8 +47,12 @@
 import org.opensearch.repositories.s3.async.AsyncExecutorContainer;
 import org.opensearch.repositories.s3.async.AsyncTransferEventLoopGroup;
 import org.opensearch.repositories.s3.async.AsyncTransferManager;
+import org.opensearch.repositories.s3.async.SizeBasedBlockingQ;
+import org.opensearch.repositories.s3.async.TransferSemaphoresHolder;
 import org.opensearch.test.OpenSearchTestCase;
+import org.opensearch.threadpool.Scheduler;
 import org.junit.After;
+import org.junit.Assert;
 import org.junit.Before;
 
 import java.io.IOException;
@@ -56,6 +60,7 @@
 import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.concurrent.CompletableFuture;
@@ -63,6 +68,7 @@
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicLong;
@@ -74,6 +80,7 @@
 
 import static org.opensearch.repositories.s3.S3Repository.BULK_DELETE_SIZE;
 import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.anyMap;
 import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.Mockito.any;
 import static org.mockito.Mockito.doAnswer;
@@ -88,8 +95,13 @@ public class S3BlobContainerMockClientTests extends OpenSearchTestCase implement
     private MockS3AsyncService asyncService;
     private ExecutorService futureCompletionService;
     private ExecutorService streamReaderService;
+    private ExecutorService remoteTransferRetry;
+    private ExecutorService transferQueueConsumerService;
+    private ScheduledExecutorService scheduler;
     private AsyncTransferEventLoopGroup transferNIOGroup;
     private S3BlobContainer blobContainer;
+    private SizeBasedBlockingQ normalPrioritySizeBasedBlockingQ;
+    private SizeBasedBlockingQ lowPrioritySizeBasedBlockingQ;
 
     static class MockS3AsyncService extends S3AsyncService {
 
@@ -361,7 +373,27 @@ public void setUp() throws Exception {
         asyncService = new MockS3AsyncService(configPath(), 1000);
         futureCompletionService = Executors.newSingleThreadExecutor();
         streamReaderService = Executors.newSingleThreadExecutor();
+        remoteTransferRetry = Executors.newFixedThreadPool(20);
+        transferQueueConsumerService = Executors.newFixedThreadPool(20);
+        scheduler = new Scheduler.SafeScheduledThreadPoolExecutor(1);
         transferNIOGroup = new AsyncTransferEventLoopGroup(1);
+        GenericStatsMetricPublisher genericStatsMetricPublisher = new GenericStatsMetricPublisher(10000L, 10, 10000L, 10);
+        normalPrioritySizeBasedBlockingQ = new SizeBasedBlockingQ(
+            new ByteSizeValue(Runtime.getRuntime().availableProcessors() * 10L, ByteSizeUnit.GB),
+            transferQueueConsumerService,
+            10,
+            genericStatsMetricPublisher,
+            SizeBasedBlockingQ.QueueEventType.NORMAL
+        );
+        lowPrioritySizeBasedBlockingQ = new SizeBasedBlockingQ(
+            new ByteSizeValue(Runtime.getRuntime().availableProcessors() * 20L, ByteSizeUnit.GB),
+            transferQueueConsumerService,
+            5,
+            genericStatsMetricPublisher,
+            SizeBasedBlockingQ.QueueEventType.NORMAL
+        );
+        normalPrioritySizeBasedBlockingQ.start();
+        lowPrioritySizeBasedBlockingQ.start();
         blobContainer = createBlobContainer();
         super.setUp();
     }
@@ -370,6 +402,14 @@ public void setUp() throws Exception {
     @After
     public void tearDown() throws Exception {
         IOUtils.close(asyncService);
+        futureCompletionService.shutdown();
+        streamReaderService.shutdown();
+        remoteTransferRetry.shutdown();
+        transferQueueConsumerService.shutdown();
+        normalPrioritySizeBasedBlockingQ.close();
+        lowPrioritySizeBasedBlockingQ.close();
+        scheduler.shutdown();
+        transferNIOGroup.close();
         super.tearDown();
     }
 
@@ -391,7 +431,7 @@ private S3BlobStore createBlobStore() {
             streamReaderService,
             transferNIOGroup
         );
-
+        GenericStatsMetricPublisher genericStatsMetricPublisher = new GenericStatsMetricPublisher(10000L, 10, 10000L, 10);
         return new S3BlobStore(
             null,
             asyncService,
@@ -407,11 +447,21 @@ private S3BlobStore createBlobStore() {
                 S3Repository.PARALLEL_MULTIPART_UPLOAD_MINIMUM_PART_SIZE_SETTING.getDefault(Settings.EMPTY).getBytes(),
                 asyncExecutorContainer.getStreamReader(),
                 asyncExecutorContainer.getStreamReader(),
-                asyncExecutorContainer.getStreamReader()
+                asyncExecutorContainer.getStreamReader(),
+                new TransferSemaphoresHolder(
+                    3,
+                    Math.max(Runtime.getRuntime().availableProcessors() * 5, 10),
+                    5,
+                    TimeUnit.MINUTES,
+                    genericStatsMetricPublisher
+                )
             ),
             asyncExecutorContainer,
             asyncExecutorContainer,
-            asyncExecutorContainer
+            asyncExecutorContainer,
+            normalPrioritySizeBasedBlockingQ,
+            lowPrioritySizeBasedBlockingQ,
+            genericStatsMetricPublisher
         );
     }
 
@@ -466,24 +516,30 @@ private void testWriteBlobByStreams(boolean expectException, boolean throwExcept
             exceptionRef.set(ex);
             countDownLatch.countDown();
         });
-        blobContainer.asyncBlobUpload(new WriteContext("write_blob_by_streams_max_retries", new StreamContextSupplier() {
-            @Override
-            public StreamContext supplyStreamContext(long partSize) {
-                return new StreamContext(new CheckedTriFunction<Integer, Long, Long, InputStreamContainer, IOException>() {
-                    @Override
-                    public InputStreamContainer apply(Integer partNo, Long size, Long position) throws IOException {
-                        InputStream inputStream = new OffsetRangeIndexInputStream(new ByteArrayIndexInput("desc", bytes), size, position);
-                        openInputStreams.add(inputStream);
-                        return new InputStreamContainer(inputStream, size, position);
-                    }
-                }, partSize, calculateLastPartSize(bytes.length, partSize), calculateNumberOfParts(bytes.length, partSize));
-            }
-        }, bytes.length, false, WritePriority.NORMAL, uploadSuccess -> {
+
+        StreamContextSupplier streamContextSupplier = partSize -> new StreamContext((partNo, size, position) -> {
+            InputStream inputStream = new OffsetRangeIndexInputStream(new ByteArrayIndexInput("desc", bytes), size, position);
+            openInputStreams.add(inputStream);
+            return new InputStreamContainer(inputStream, size, position);
+        }, partSize, calculateLastPartSize(bytes.length, partSize), calculateNumberOfParts(bytes.length, partSize));
+
+        CheckedConsumer<Boolean, IOException> uploadFinalizer = uploadSuccess -> {
             assertTrue(uploadSuccess);
             if (throwExceptionOnFinalizeUpload) {
                 throw new RuntimeException();
             }
-        }, false, null), completionListener);
+        };
+
+        WriteContext writeContext = new WriteContext.Builder().fileName("write_blob_by_streams_max_retries")
+            .streamContextSupplier(streamContextSupplier)
+            .fileSize(bytes.length)
+            .failIfAlreadyExists(false)
+            .writePriority(WritePriority.NORMAL)
+            .uploadFinalizer(uploadFinalizer)
+            .doRemoteDataIntegrityCheck(false)
+            .build();
+
+        blobContainer.asyncBlobUpload(writeContext, completionListener);
 
         assertTrue(countDownLatch.await(5000, TimeUnit.SECONDS));
         // wait for completableFuture to finish
@@ -516,24 +572,30 @@ private void testWriteBlobByStreamsLargeBlob(boolean expectException, boolean th
             countDownLatch.countDown();
         });
         List<InputStream> openInputStreams = new ArrayList<>();
-        blobContainer.asyncBlobUpload(new WriteContext("write_large_blob", new StreamContextSupplier() {
-            @Override
-            public StreamContext supplyStreamContext(long partSize) {
-                return new StreamContext(new CheckedTriFunction<Integer, Long, Long, InputStreamContainer, IOException>() {
-                    @Override
-                    public InputStreamContainer apply(Integer partNo, Long size, Long position) throws IOException {
-                        InputStream inputStream = new OffsetRangeIndexInputStream(new ZeroIndexInput("desc", blobSize), size, position);
-                        openInputStreams.add(inputStream);
-                        return new InputStreamContainer(inputStream, size, position);
-                    }
-                }, partSize, calculateLastPartSize(blobSize, partSize), calculateNumberOfParts(blobSize, partSize));
-            }
-        }, blobSize, false, WritePriority.NORMAL, uploadSuccess -> {
+
+        StreamContextSupplier streamContextSupplier = partSize1 -> new StreamContext((partNo, size, position) -> {
+            InputStream inputStream = new OffsetRangeIndexInputStream(new ZeroIndexInput("desc", blobSize), size, position);
+            openInputStreams.add(inputStream);
+            return new InputStreamContainer(inputStream, size, position);
+        }, partSize1, calculateLastPartSize(blobSize, partSize1), calculateNumberOfParts(blobSize, partSize1));
+
+        CheckedConsumer<Boolean, IOException> uploadFinalizer = uploadSuccess -> {
             assertTrue(uploadSuccess);
             if (throwExceptionOnFinalizeUpload) {
                 throw new RuntimeException();
             }
-        }, false, null), completionListener);
+        };
+
+        WriteContext writeContext = new WriteContext.Builder().fileName("write_large_blob")
+            .streamContextSupplier(streamContextSupplier)
+            .fileSize(blobSize)
+            .failIfAlreadyExists(false)
+            .writePriority(WritePriority.NORMAL)
+            .uploadFinalizer(uploadFinalizer)
+            .doRemoteDataIntegrityCheck(false)
+            .build();
+
+        blobContainer.asyncBlobUpload(writeContext, completionListener);
 
         assertTrue(countDownLatch.await(5000, TimeUnit.SECONDS));
         if (expectException || throwExceptionOnFinalizeUpload) {
@@ -559,19 +621,32 @@ private int calculateNumberOfParts(long contentLength, long partSize) {
         return (int) ((contentLength % partSize) == 0 ? contentLength / partSize : (contentLength / partSize) + 1);
     }
 
-    public void testFailureWhenLargeFileRedirected() throws IOException, ExecutionException, InterruptedException {
-        testLargeFilesRedirectedToSlowSyncClient(true);
+    public void testFailureWhenLargeFileRedirected() throws IOException, InterruptedException {
+        testLargeFilesRedirectedToSlowSyncClient(true, WritePriority.LOW);
+        testLargeFilesRedirectedToSlowSyncClient(true, WritePriority.NORMAL);
     }
 
-    public void testLargeFileRedirected() throws IOException, ExecutionException, InterruptedException {
-        testLargeFilesRedirectedToSlowSyncClient(false);
+    public void testLargeFileRedirected() throws IOException, InterruptedException {
+        testLargeFilesRedirectedToSlowSyncClient(false, WritePriority.LOW);
+        testLargeFilesRedirectedToSlowSyncClient(false, WritePriority.NORMAL);
     }
 
-    private void testLargeFilesRedirectedToSlowSyncClient(boolean expectException) throws IOException, InterruptedException {
-        final ByteSizeValue partSize = new ByteSizeValue(1024, ByteSizeUnit.MB);
-
+    private void testLargeFilesRedirectedToSlowSyncClient(boolean expectException, WritePriority writePriority) throws IOException,
+        InterruptedException {
+        ByteSizeValue capacity = new ByteSizeValue(1, ByteSizeUnit.GB);
         int numberOfParts = 20;
-        final long lastPartSize = new ByteSizeValue(20, ByteSizeUnit.MB).getBytes();
+        final ByteSizeValue partSize = new ByteSizeValue(capacity.getBytes() / numberOfParts + 1, ByteSizeUnit.BYTES);
+
+        GenericStatsMetricPublisher genericStatsMetricPublisher = new GenericStatsMetricPublisher(10000L, 10, 10000L, 10);
+        SizeBasedBlockingQ sizeBasedBlockingQ = new SizeBasedBlockingQ(
+            capacity,
+            transferQueueConsumerService,
+            10,
+            genericStatsMetricPublisher,
+            SizeBasedBlockingQ.QueueEventType.NORMAL
+        );
+
+        final long lastPartSize = new ByteSizeValue(200, ByteSizeUnit.MB).getBytes();
         final long blobSize = ((numberOfParts - 1) * partSize.getBytes()) + lastPartSize;
         CountDownLatch countDownLatch = new CountDownLatch(1);
         AtomicReference<Exception> exceptionRef = new AtomicReference<>();
@@ -594,6 +669,9 @@ private void testLargeFilesRedirectedToSlowSyncClient(boolean expectException) t
         when(blobStore.getStatsMetricPublisher()).thenReturn(new StatsMetricPublisher());
         when(blobStore.bufferSizeInBytes()).thenReturn(bufferSize);
 
+        when(blobStore.getLowPrioritySizeBasedBlockingQ()).thenReturn(sizeBasedBlockingQ);
+        when(blobStore.getNormalPrioritySizeBasedBlockingQ()).thenReturn(sizeBasedBlockingQ);
+
         final boolean serverSideEncryption = randomBoolean();
         when(blobStore.serverSideEncryption()).thenReturn(serverSideEncryption);
 
@@ -632,27 +710,37 @@ private void testLargeFilesRedirectedToSlowSyncClient(boolean expectException) t
 
         List<InputStream> openInputStreams = new ArrayList<>();
         final S3BlobContainer s3BlobContainer = Mockito.spy(new S3BlobContainer(blobPath, blobStore));
-        s3BlobContainer.asyncBlobUpload(new WriteContext("write_large_blob", new StreamContextSupplier() {
-            @Override
-            public StreamContext supplyStreamContext(long partSize) {
-                return new StreamContext(new CheckedTriFunction<Integer, Long, Long, InputStreamContainer, IOException>() {
-                    @Override
-                    public InputStreamContainer apply(Integer partNo, Long size, Long position) throws IOException {
-                        InputStream inputStream = new OffsetRangeIndexInputStream(new ZeroIndexInput("desc", blobSize), size, position);
-                        openInputStreams.add(inputStream);
-                        return new InputStreamContainer(inputStream, size, position);
-                    }
-                }, partSize, calculateLastPartSize(blobSize, partSize), calculateNumberOfParts(blobSize, partSize));
-            }
-        }, blobSize, false, WritePriority.HIGH, uploadSuccess -> { assertTrue(uploadSuccess); }, false, null), completionListener);
 
+        StreamContextSupplier streamContextSupplier = partSize1 -> new StreamContext((partNo, size, position) -> {
+            InputStream inputStream = new OffsetRangeIndexInputStream(new ZeroIndexInput("desc", blobSize), size, position);
+            openInputStreams.add(inputStream);
+            return new InputStreamContainer(inputStream, size, position);
+        }, partSize1, calculateLastPartSize(blobSize, partSize1), calculateNumberOfParts(blobSize, partSize1));
+
+        WriteContext writeContext = new WriteContext.Builder().fileName("write_large_blob")
+            .streamContextSupplier(streamContextSupplier)
+            .fileSize(blobSize)
+            .failIfAlreadyExists(false)
+            .writePriority(writePriority)
+            .uploadFinalizer(Assert::assertTrue)
+            .doRemoteDataIntegrityCheck(false)
+            .metadata(new HashMap<>())
+            .build();
+
+        s3BlobContainer.asyncBlobUpload(writeContext, completionListener);
         assertTrue(countDownLatch.await(5000, TimeUnit.SECONDS));
         if (expectException) {
             assertNotNull(exceptionRef.get());
         } else {
             assertNull(exceptionRef.get());
         }
-        verify(s3BlobContainer, times(1)).executeMultipartUpload(any(S3BlobStore.class), anyString(), any(InputStream.class), anyLong());
+        verify(s3BlobContainer, times(1)).executeMultipartUpload(
+            any(S3BlobStore.class),
+            anyString(),
+            any(InputStream.class),
+            anyLong(),
+            anyMap()
+        );
 
         if (expectException) {
             verify(client, times(1)).abortMultipartUpload(any(AbortMultipartUploadRequest.class));
@@ -668,5 +756,4 @@ public InputStreamContainer apply(Integer partNo, Long size, Long position) thro
             }
         });
     }
-
 }
diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java
index ceab06bd051e9..96ef28d24c14f 100644
--- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java
+++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobContainerRetriesTests.java
@@ -37,7 +37,6 @@
 
 import org.apache.http.HttpStatus;
 import org.opensearch.cluster.metadata.RepositoryMetadata;
-import org.opensearch.common.CheckedTriFunction;
 import org.opensearch.common.Nullable;
 import org.opensearch.common.StreamContext;
 import org.opensearch.common.SuppressForbidden;
@@ -68,6 +67,8 @@
 import org.opensearch.repositories.s3.async.AsyncExecutorContainer;
 import org.opensearch.repositories.s3.async.AsyncTransferEventLoopGroup;
 import org.opensearch.repositories.s3.async.AsyncTransferManager;
+import org.opensearch.repositories.s3.async.SizeBasedBlockingQ;
+import org.opensearch.repositories.s3.async.TransferSemaphoresHolder;
 import org.junit.After;
 import org.junit.Assert;
 import org.junit.Before;
@@ -80,12 +81,16 @@
 import java.net.SocketTimeoutException;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
+import java.util.Map;
 import java.util.Objects;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
@@ -113,7 +118,12 @@ public class S3BlobContainerRetriesTests extends AbstractBlobContainerRetriesTes
     private S3AsyncService asyncService;
     private ExecutorService futureCompletionService;
     private ExecutorService streamReaderService;
+    private ExecutorService remoteTransferRetry;
+    private ExecutorService transferQueueConsumerService;
+    private ScheduledExecutorService scheduler;
     private AsyncTransferEventLoopGroup transferNIOGroup;
+    private SizeBasedBlockingQ normalPrioritySizeBasedBlockingQ;
+    private SizeBasedBlockingQ lowPrioritySizeBasedBlockingQ;
 
     @Before
     public void setUp() throws Exception {
@@ -124,7 +134,26 @@ public void setUp() throws Exception {
         futureCompletionService = Executors.newSingleThreadExecutor();
         streamReaderService = Executors.newSingleThreadExecutor();
         transferNIOGroup = new AsyncTransferEventLoopGroup(1);
-
+        remoteTransferRetry = Executors.newFixedThreadPool(20);
+        transferQueueConsumerService = Executors.newFixedThreadPool(2);
+        scheduler = new ScheduledThreadPoolExecutor(1);
+        GenericStatsMetricPublisher genericStatsMetricPublisher = new GenericStatsMetricPublisher(10000L, 10, 10000L, 10);
+        normalPrioritySizeBasedBlockingQ = new SizeBasedBlockingQ(
+            new ByteSizeValue(Runtime.getRuntime().availableProcessors() * 5L, ByteSizeUnit.GB),
+            transferQueueConsumerService,
+            2,
+            genericStatsMetricPublisher,
+            SizeBasedBlockingQ.QueueEventType.NORMAL
+        );
+        lowPrioritySizeBasedBlockingQ = new SizeBasedBlockingQ(
+            new ByteSizeValue(Runtime.getRuntime().availableProcessors() * 5L, ByteSizeUnit.GB),
+            transferQueueConsumerService,
+            2,
+            genericStatsMetricPublisher,
+            SizeBasedBlockingQ.QueueEventType.LOW
+        );
+        normalPrioritySizeBasedBlockingQ.start();
+        lowPrioritySizeBasedBlockingQ.start();
         // needed by S3AsyncService
         SocketAccess.doPrivileged(() -> System.setProperty("opensearch.path.conf", configPath().toString()));
         super.setUp();
@@ -136,6 +165,11 @@ public void tearDown() throws Exception {
 
         streamReaderService.shutdown();
         futureCompletionService.shutdown();
+        remoteTransferRetry.shutdown();
+        transferQueueConsumerService.shutdown();
+        scheduler.shutdown();
+        normalPrioritySizeBasedBlockingQ.close();
+        lowPrioritySizeBasedBlockingQ.close();
         IOUtils.close(transferNIOGroup);
 
         if (previousOpenSearchPathConf != null) {
@@ -204,7 +238,7 @@ protected AsyncMultiStreamBlobContainer createBlobContainer(
             streamReaderService,
             transferNIOGroup
         );
-
+        GenericStatsMetricPublisher genericStatsMetricPublisher = new GenericStatsMetricPublisher(10000L, 10, 10000L, 10);
         return new S3BlobContainer(
             BlobPath.cleanPath(),
             new S3BlobStore(
@@ -222,11 +256,21 @@ protected AsyncMultiStreamBlobContainer createBlobContainer(
                     S3Repository.PARALLEL_MULTIPART_UPLOAD_MINIMUM_PART_SIZE_SETTING.getDefault(Settings.EMPTY).getBytes(),
                     asyncExecutorContainer.getStreamReader(),
                     asyncExecutorContainer.getStreamReader(),
-                    asyncExecutorContainer.getStreamReader()
+                    asyncExecutorContainer.getStreamReader(),
+                    new TransferSemaphoresHolder(
+                        3,
+                        Math.max(Runtime.getRuntime().availableProcessors() * 5, 10),
+                        5,
+                        TimeUnit.MINUTES,
+                        genericStatsMetricPublisher
+                    )
                 ),
                 asyncExecutorContainer,
                 asyncExecutorContainer,
-                asyncExecutorContainer
+                asyncExecutorContainer,
+                normalPrioritySizeBasedBlockingQ,
+                lowPrioritySizeBasedBlockingQ,
+                genericStatsMetricPublisher
             )
         ) {
             @Override
@@ -241,7 +285,7 @@ public InputStream readBlob(String blobName, long position, long length) throws
         };
     }
 
-    public void testWriteBlobWithRetries() throws Exception {
+    public void writeBlobWithRetriesHelper(Map<String, String> metadata) throws Exception {
         final int maxRetries = randomInt(5);
         final CountDown countDown = new CountDown(maxRetries + 1);
 
@@ -281,11 +325,26 @@ public void testWriteBlobWithRetries() throws Exception {
 
         final BlobContainer blobContainer = createBlobContainer(maxRetries, null, true, null);
         try (InputStream stream = new ByteArrayInputStream(bytes)) {
-            blobContainer.writeBlob("write_blob_max_retries", stream, bytes.length, false);
+            if (metadata != null) {
+                blobContainer.writeBlobWithMetadata("write_blob_max_retries", stream, bytes.length, false, metadata);
+            } else {
+                blobContainer.writeBlob("write_blob_max_retries", stream, bytes.length, false);
+            }
         }
         assertThat(countDown.isCountedDown(), is(true));
     }
 
+    public void testWriteBlobWithMetadataWithRetries() throws Exception {
+        Map<String, String> metadata = new HashMap<>();
+        metadata.put("key1", "value1");
+        metadata.put("key2", "value2");
+        writeBlobWithRetriesHelper(metadata);
+    }
+
+    public void testWriteBlobWithRetries() throws Exception {
+        writeBlobWithRetriesHelper(null);
+    }
+
     public void testWriteBlobByStreamsWithRetries() throws Exception {
         final int maxRetries = randomInt(5);
         final CountDown countDown = new CountDown(maxRetries + 1);
@@ -332,22 +391,24 @@ public void testWriteBlobByStreamsWithRetries() throws Exception {
             exceptionRef.set(ex);
             countDownLatch.countDown();
         });
-        blobContainer.asyncBlobUpload(new WriteContext("write_blob_by_streams_max_retries", new StreamContextSupplier() {
-            @Override
-            public StreamContext supplyStreamContext(long partSize) {
-                return new StreamContext(new CheckedTriFunction<Integer, Long, Long, InputStreamContainer, IOException>() {
-                    @Override
-                    public InputStreamContainer apply(Integer partNo, Long size, Long position) throws IOException {
-                        InputStream inputStream = new OffsetRangeIndexInputStream(new ByteArrayIndexInput("desc", bytes), size, position);
-                        openInputStreams.add(inputStream);
-                        return new InputStreamContainer(inputStream, size, position);
-                    }
-                }, partSize, calculateLastPartSize(bytes.length, partSize), calculateNumberOfParts(bytes.length, partSize));
-            }
-        }, bytes.length, false, WritePriority.NORMAL, Assert::assertTrue, false, null), completionListener);
 
+        StreamContextSupplier streamContextSupplier = partSize -> new StreamContext((partNo, size, position) -> {
+            InputStream inputStream = new OffsetRangeIndexInputStream(new ByteArrayIndexInput("desc", bytes), size, position);
+            openInputStreams.add(inputStream);
+            return new InputStreamContainer(inputStream, size, position);
+        }, partSize, calculateLastPartSize(bytes.length, partSize), calculateNumberOfParts(bytes.length, partSize));
+
+        WriteContext writeContext = new WriteContext.Builder().fileName("write_blob_by_streams_max_retries")
+            .streamContextSupplier(streamContextSupplier)
+            .fileSize(bytes.length)
+            .failIfAlreadyExists(false)
+            .writePriority(WritePriority.NORMAL)
+            .uploadFinalizer(Assert::assertTrue)
+            .doRemoteDataIntegrityCheck(false)
+            .build();
+
+        blobContainer.asyncBlobUpload(writeContext, completionListener);
         assertTrue(countDownLatch.await(5000, TimeUnit.SECONDS));
-
         assertThat(countDown.isCountedDown(), is(true));
 
         openInputStreams.forEach(inputStream -> {
@@ -367,7 +428,7 @@ private int calculateNumberOfParts(long contentLength, long partSize) {
         return (int) ((contentLength % partSize) == 0 ? contentLength / partSize : (contentLength / partSize) + 1);
     }
 
-    public void testWriteBlobWithReadTimeouts() {
+    public void writeBlobWithReadTimeoutsHelper(Map<String, String> metadata) {
         final byte[] bytes = randomByteArrayOfLength(randomIntBetween(10, 128));
         final TimeValue readTimeout = TimeValue.timeValueMillis(randomIntBetween(100, 500));
         final BlobContainer blobContainer = createBlobContainer(1, readTimeout, true, null);
@@ -385,7 +446,11 @@ public void testWriteBlobWithReadTimeouts() {
 
         Exception exception = expectThrows(IOException.class, () -> {
             try (InputStream stream = new InputStreamIndexInput(new ByteArrayIndexInput("desc", bytes), bytes.length)) {
-                blobContainer.writeBlob("write_blob_timeout", stream, bytes.length, false);
+                if (metadata != null) {
+                    blobContainer.writeBlobWithMetadata("write_blob_timeout", stream, bytes.length, false, metadata);
+                } else {
+                    blobContainer.writeBlob("write_blob_timeout", stream, bytes.length, false);
+                }
             }
         });
         assertThat(
@@ -400,7 +465,18 @@ public void testWriteBlobWithReadTimeouts() {
         assertThat(exception.getCause().getCause().getMessage().toLowerCase(Locale.ROOT), containsString("read timed out"));
     }
 
-    public void testWriteLargeBlob() throws Exception {
+    public void testWriteBlobWithMetadataWithReadTimeouts() throws Exception {
+        Map<String, String> metadata = new HashMap<>();
+        metadata.put("key1", "value1");
+        metadata.put("key2", "value2");
+        writeBlobWithReadTimeoutsHelper(metadata);
+    }
+
+    public void testWriteBlobWithReadTimeouts() throws Exception {
+        writeBlobWithReadTimeoutsHelper(null);
+    }
+
+    public void WriteLargeBlobHelper(Map<String, String> metadata) throws Exception {
         final boolean useTimeout = rarely();
         final TimeValue readTimeout = useTimeout ? TimeValue.timeValueMillis(randomIntBetween(100, 500)) : null;
         final ByteSizeValue bufferSize = new ByteSizeValue(5, ByteSizeUnit.MB);
@@ -486,13 +562,28 @@ public void testWriteLargeBlob() throws Exception {
             }
         });
 
-        blobContainer.writeBlob("write_large_blob", new ZeroInputStream(blobSize), blobSize, false);
+        if (metadata != null) {
+            blobContainer.writeBlobWithMetadata("write_large_blob", new ZeroInputStream(blobSize), blobSize, false, metadata);
+        } else {
+            blobContainer.writeBlob("write_large_blob", new ZeroInputStream(blobSize), blobSize, false);
+        }
 
         assertThat(countDownInitiate.isCountedDown(), is(true));
         assertThat(countDownUploads.get(), equalTo(0));
         assertThat(countDownComplete.isCountedDown(), is(true));
     }
 
+    public void testWriteLargeBlobWithMetadata() throws Exception {
+        Map<String, String> metadata = new HashMap<>();
+        metadata.put("key1", "value1");
+        metadata.put("key2", "value2");
+        WriteLargeBlobHelper(metadata);
+    }
+
+    public void testWriteLargeBlob() throws Exception {
+        WriteLargeBlobHelper(null);
+    }
+
     /**
      * Asserts that an InputStream is fully consumed, or aborted, when it is closed
      */
diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java
index 2b45e9cfe2d4b..654d8a72690c4 100644
--- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java
+++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3BlobStoreContainerTests.java
@@ -90,6 +90,7 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
@@ -125,7 +126,7 @@ public void testExecuteSingleUploadBlobSizeTooLarge() {
 
         final IllegalArgumentException e = expectThrows(
             IllegalArgumentException.class,
-            () -> blobContainer.executeSingleUpload(blobStore, randomAlphaOfLengthBetween(1, 10), null, blobSize)
+            () -> blobContainer.executeSingleUpload(blobStore, randomAlphaOfLengthBetween(1, 10), null, blobSize, null)
         );
         assertEquals("Upload request size [" + blobSize + "] can't be larger than 5gb", e.getMessage());
     }
@@ -139,7 +140,13 @@ public void testExecuteSingleUploadBlobSizeLargerThanBufferSize() {
 
         final IllegalArgumentException e = expectThrows(
             IllegalArgumentException.class,
-            () -> blobContainer.executeSingleUpload(blobStore, blobName, new ByteArrayInputStream(new byte[0]), ByteSizeUnit.MB.toBytes(2))
+            () -> blobContainer.executeSingleUpload(
+                blobStore,
+                blobName,
+                new ByteArrayInputStream(new byte[0]),
+                ByteSizeUnit.MB.toBytes(2),
+                null
+            )
         );
         assertEquals("Upload request size [2097152] can't be larger than buffer size", e.getMessage());
     }
@@ -430,6 +437,10 @@ public void testExecuteSingleUpload() throws IOException {
         final String bucketName = randomAlphaOfLengthBetween(1, 10);
         final String blobName = randomAlphaOfLengthBetween(1, 10);
 
+        final Map<String, String> metadata = new HashMap<>();
+        metadata.put("key1", "value1");
+        metadata.put("key2", "value2");
+
         final BlobPath blobPath = new BlobPath();
         if (randomBoolean()) {
             IntStream.of(randomIntBetween(1, 5)).forEach(value -> blobPath.add("path_" + value));
@@ -467,7 +478,7 @@ public void testExecuteSingleUpload() throws IOException {
         );
 
         final ByteArrayInputStream inputStream = new ByteArrayInputStream(new byte[blobSize]);
-        blobContainer.executeSingleUpload(blobStore, blobName, inputStream, blobSize);
+        blobContainer.executeSingleUpload(blobStore, blobName, inputStream, blobSize, metadata);
 
         final PutObjectRequest request = putObjectRequestArgumentCaptor.getValue();
         final RequestBody requestBody = requestBodyArgumentCaptor.getValue();
@@ -480,6 +491,7 @@ public void testExecuteSingleUpload() throws IOException {
         assertEquals(blobSize, request.contentLength().longValue());
         assertEquals(storageClass, request.storageClass());
         assertEquals(cannedAccessControlList, request.acl());
+        assertEquals(metadata, request.metadata());
         if (serverSideEncryption) {
             assertEquals(ServerSideEncryption.AES256, request.serverSideEncryption());
         }
@@ -492,7 +504,7 @@ public void testExecuteMultipartUploadBlobSizeTooLarge() {
 
         final IllegalArgumentException e = expectThrows(
             IllegalArgumentException.class,
-            () -> blobContainer.executeMultipartUpload(blobStore, randomAlphaOfLengthBetween(1, 10), null, blobSize)
+            () -> blobContainer.executeMultipartUpload(blobStore, randomAlphaOfLengthBetween(1, 10), null, blobSize, null)
         );
         assertEquals("Multipart upload request size [" + blobSize + "] can't be larger than 5tb", e.getMessage());
     }
@@ -504,7 +516,7 @@ public void testExecuteMultipartUploadBlobSizeTooSmall() {
 
         final IllegalArgumentException e = expectThrows(
             IllegalArgumentException.class,
-            () -> blobContainer.executeMultipartUpload(blobStore, randomAlphaOfLengthBetween(1, 10), null, blobSize)
+            () -> blobContainer.executeMultipartUpload(blobStore, randomAlphaOfLengthBetween(1, 10), null, blobSize, null)
         );
         assertEquals("Multipart upload request size [" + blobSize + "] can't be smaller than 5mb", e.getMessage());
     }
@@ -513,6 +525,10 @@ public void testExecuteMultipartUpload() throws IOException {
         final String bucketName = randomAlphaOfLengthBetween(1, 10);
         final String blobName = randomAlphaOfLengthBetween(1, 10);
 
+        final Map<String, String> metadata = new HashMap<>();
+        metadata.put("key1", "value1");
+        metadata.put("key2", "value2");
+
         final BlobPath blobPath = new BlobPath();
         if (randomBoolean()) {
             IntStream.of(randomIntBetween(1, 5)).forEach(value -> blobPath.add("path_" + value));
@@ -577,13 +593,15 @@ public void testExecuteMultipartUpload() throws IOException {
 
         final ByteArrayInputStream inputStream = new ByteArrayInputStream(new byte[0]);
         final S3BlobContainer blobContainer = new S3BlobContainer(blobPath, blobStore);
-        blobContainer.executeMultipartUpload(blobStore, blobName, inputStream, blobSize);
+        blobContainer.executeMultipartUpload(blobStore, blobName, inputStream, blobSize, metadata);
 
         final CreateMultipartUploadRequest initRequest = createMultipartUploadRequestArgumentCaptor.getValue();
         assertEquals(bucketName, initRequest.bucket());
         assertEquals(blobPath.buildAsString() + blobName, initRequest.key());
         assertEquals(storageClass, initRequest.storageClass());
         assertEquals(cannedAccessControlList, initRequest.acl());
+        assertEquals(metadata, initRequest.metadata());
+
         if (serverSideEncryption) {
             assertEquals(ServerSideEncryption.AES256, initRequest.serverSideEncryption());
         }
@@ -686,7 +704,7 @@ public void testExecuteMultipartUploadAborted() {
 
         final IOException e = expectThrows(IOException.class, () -> {
             final S3BlobContainer blobContainer = new S3BlobContainer(blobPath, blobStore);
-            blobContainer.executeMultipartUpload(blobStore, blobName, new ByteArrayInputStream(new byte[0]), blobSize);
+            blobContainer.executeMultipartUpload(blobStore, blobName, new ByteArrayInputStream(new byte[0]), blobSize, null);
         });
 
         assertEquals("Unable to upload object [" + blobName + "] using multipart upload", e.getMessage());
diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryTests.java
index 6fec535ae6301..f8e9903bb3577 100644
--- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryTests.java
+++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryTests.java
@@ -169,7 +169,11 @@ private S3Repository createS3Repo(RepositoryMetadata metadata) {
             null,
             null,
             null,
-            false
+            false,
+            null,
+            null,
+            null,
+            null
         ) {
             @Override
             protected void assertSnapshotOrGenericThread() {
diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RetryingInputStreamTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RetryingInputStreamTests.java
index b38d5119b4108..d884a46f3ecc5 100644
--- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RetryingInputStreamTests.java
+++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RetryingInputStreamTests.java
@@ -43,6 +43,8 @@
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
 
 import static org.hamcrest.Matchers.is;
 import static org.mockito.Mockito.any;
@@ -61,6 +63,15 @@ public void testInputStreamFullyConsumed() throws IOException {
         assertThat(stream.isAborted(), is(false));
     }
 
+    public void testInputStreamGetMetadata() throws IOException {
+        final byte[] expectedBytes = randomByteArrayOfLength(randomIntBetween(1, 512));
+
+        final S3RetryingInputStream stream = createInputStream(expectedBytes, 0L, (long) (Integer.MAX_VALUE - 1));
+
+        Map<String, String> metadata = new HashMap<>();
+        assertEquals(stream.getMetadata(), metadata);
+    }
+
     public void testInputStreamIsAborted() throws IOException {
         final byte[] expectedBytes = randomByteArrayOfLength(randomIntBetween(10, 512));
         final byte[] actualBytes = new byte[randomIntBetween(1, Math.max(1, expectedBytes.length - 1))];
diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/AsyncTransferManagerTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/AsyncTransferManagerTests.java
index b753b847df869..89add3cdbfc60 100644
--- a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/AsyncTransferManagerTests.java
+++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/AsyncTransferManagerTests.java
@@ -33,6 +33,7 @@
 import org.opensearch.common.io.InputStreamContainer;
 import org.opensearch.core.common.unit.ByteSizeUnit;
 import org.opensearch.repositories.blobstore.ZeroInputStream;
+import org.opensearch.repositories.s3.GenericStatsMetricPublisher;
 import org.opensearch.repositories.s3.StatsMetricPublisher;
 import org.opensearch.test.OpenSearchTestCase;
 import org.junit.Before;
@@ -40,10 +41,13 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
 
 import static org.mockito.ArgumentMatchers.any;
@@ -61,11 +65,19 @@ public class AsyncTransferManagerTests extends OpenSearchTestCase {
     @Before
     public void setUp() throws Exception {
         s3AsyncClient = mock(S3AsyncClient.class);
+        GenericStatsMetricPublisher genericStatsMetricPublisher = new GenericStatsMetricPublisher(10000L, 10, 10000L, 10);
         asyncTransferManager = new AsyncTransferManager(
             ByteSizeUnit.MB.toBytes(5),
             Executors.newSingleThreadExecutor(),
             Executors.newSingleThreadExecutor(),
-            Executors.newSingleThreadExecutor()
+            Executors.newSingleThreadExecutor(),
+            new TransferSemaphoresHolder(
+                3,
+                Math.max(Runtime.getRuntime().availableProcessors() * 5, 10),
+                5,
+                TimeUnit.MINUTES,
+                genericStatsMetricPublisher
+            )
         );
         super.setUp();
     }
@@ -78,11 +90,14 @@ public void testOneChunkUpload() {
         );
 
         AtomicReference<InputStream> streamRef = new AtomicReference<>();
+        Map<String, String> metadata = new HashMap<>();
+        metadata.put("key1", "value1");
+        metadata.put("key2", "value2");
         CompletableFuture<Void> resultFuture = asyncTransferManager.uploadObject(
             s3AsyncClient,
             new UploadRequest("bucket", "key", ByteSizeUnit.MB.toBytes(1), WritePriority.HIGH, uploadSuccess -> {
                 // do nothing
-            }, false, null, true),
+            }, false, null, true, metadata),
             new StreamContext((partIdx, partSize, position) -> {
                 streamRef.set(new ZeroInputStream(partSize));
                 return new InputStreamContainer(streamRef.get(), partSize, position);
@@ -123,11 +138,15 @@ public void testOneChunkUploadCorruption() {
         deleteObjectResponseCompletableFuture.complete(DeleteObjectResponse.builder().build());
         when(s3AsyncClient.deleteObject(any(DeleteObjectRequest.class))).thenReturn(deleteObjectResponseCompletableFuture);
 
+        Map<String, String> metadata = new HashMap<>();
+        metadata.put("key1", "value1");
+        metadata.put("key2", "value2");
+
         CompletableFuture<Void> resultFuture = asyncTransferManager.uploadObject(
             s3AsyncClient,
             new UploadRequest("bucket", "key", ByteSizeUnit.MB.toBytes(1), WritePriority.HIGH, uploadSuccess -> {
                 // do nothing
-            }, false, null, true),
+            }, false, null, true, metadata),
             new StreamContext(
                 (partIdx, partSize, position) -> new InputStreamContainer(new ZeroInputStream(partSize), partSize, position),
                 ByteSizeUnit.MB.toBytes(1),
@@ -175,12 +194,16 @@ public void testMultipartUpload() {
             abortMultipartUploadResponseCompletableFuture
         );
 
+        Map<String, String> metadata = new HashMap<>();
+        metadata.put("key1", "value1");
+        metadata.put("key2", "value2");
+
         List<InputStream> streams = new ArrayList<>();
         CompletableFuture<Void> resultFuture = asyncTransferManager.uploadObject(
             s3AsyncClient,
             new UploadRequest("bucket", "key", ByteSizeUnit.MB.toBytes(5), WritePriority.HIGH, uploadSuccess -> {
                 // do nothing
-            }, true, 3376132981L, true),
+            }, true, 3376132981L, true, metadata),
             new StreamContext((partIdx, partSize, position) -> {
                 InputStream stream = new ZeroInputStream(partSize);
                 streams.add(stream);
@@ -236,11 +259,15 @@ public void testMultipartUploadCorruption() {
             abortMultipartUploadResponseCompletableFuture
         );
 
+        Map<String, String> metadata = new HashMap<>();
+        metadata.put("key1", "value1");
+        metadata.put("key2", "value2");
+
         CompletableFuture<Void> resultFuture = asyncTransferManager.uploadObject(
             s3AsyncClient,
             new UploadRequest("bucket", "key", ByteSizeUnit.MB.toBytes(5), WritePriority.HIGH, uploadSuccess -> {
                 // do nothing
-            }, true, 0L, true),
+            }, true, 0L, true, metadata),
             new StreamContext(
                 (partIdx, partSize, position) -> new InputStreamContainer(new ZeroInputStream(partSize), partSize, position),
                 ByteSizeUnit.MB.toBytes(1),
diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/SizeBasedBlockingQTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/SizeBasedBlockingQTests.java
new file mode 100644
index 0000000000000..5be4037407d23
--- /dev/null
+++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/SizeBasedBlockingQTests.java
@@ -0,0 +1,102 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.repositories.s3.async;
+
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
+import org.opensearch.repositories.s3.GenericStatsMetricPublisher;
+import org.opensearch.repositories.s3.S3TransferRejectedException;
+import org.opensearch.test.OpenSearchTestCase;
+import org.junit.After;
+import org.junit.Before;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+public class SizeBasedBlockingQTests extends OpenSearchTestCase {
+    private ExecutorService consumerService;
+    private ExecutorService producerService;
+
+    @Override
+    @Before
+    public void setUp() throws Exception {
+        this.consumerService = Executors.newFixedThreadPool(10);
+        this.producerService = Executors.newFixedThreadPool(100);
+        super.setUp();
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        consumerService.shutdown();
+        producerService.shutdown();
+        super.tearDown();
+    }
+
+    public void testProducerConsumerOfBulkItems() throws InterruptedException {
+        GenericStatsMetricPublisher genericStatsMetricPublisher = new GenericStatsMetricPublisher(10000L, 10, 10000L, 10);
+        SizeBasedBlockingQ.QueueEventType queueEventType = randomBoolean()
+            ? SizeBasedBlockingQ.QueueEventType.NORMAL
+            : SizeBasedBlockingQ.QueueEventType.LOW;
+        SizeBasedBlockingQ sizeBasedBlockingQ = new SizeBasedBlockingQ(
+            new ByteSizeValue(ByteSizeUnit.BYTES.toBytes(10)),
+            consumerService,
+            10,
+            genericStatsMetricPublisher,
+            queueEventType
+        );
+        sizeBasedBlockingQ.start();
+        int numOfItems = randomIntBetween(100, 1000);
+        CountDownLatch latch = new CountDownLatch(numOfItems);
+        AtomicBoolean unknownError = new AtomicBoolean();
+        for (int i = 0; i < numOfItems; i++) {
+            final int idx = i;
+            producerService.submit(() -> {
+                boolean throwException = randomBoolean();
+
+                SizeBasedBlockingQ.Item item = new TestItemToStr(randomIntBetween(1, 5), () -> {
+                    latch.countDown();
+                    if (throwException) {
+                        throw new RuntimeException("throwing random exception");
+                    }
+                }, idx);
+
+                try {
+                    sizeBasedBlockingQ.produce(item);
+                } catch (InterruptedException e) {
+                    latch.countDown();
+                    unknownError.set(true);
+                    throw new RuntimeException(e);
+                } catch (S3TransferRejectedException ex) {
+                    latch.countDown();
+                }
+            });
+        }
+        latch.await();
+        sizeBasedBlockingQ.close();
+        assertFalse(unknownError.get());
+        assertEquals(0L, genericStatsMetricPublisher.getNormalPriorityQSize());
+        assertEquals(0L, genericStatsMetricPublisher.getLowPriorityQSize());
+    }
+
+    static class TestItemToStr extends SizeBasedBlockingQ.Item {
+        private final int id;
+
+        public TestItemToStr(long size, Runnable consumable, int id) {
+            super(size, consumable);
+            this.id = id;
+        }
+
+        @Override
+        public String toString() {
+            return String.valueOf(id);
+        }
+    }
+}
diff --git a/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/TransferSemaphoresHolderTests.java b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/TransferSemaphoresHolderTests.java
new file mode 100644
index 0000000000000..236f02c5eb1f7
--- /dev/null
+++ b/plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/async/TransferSemaphoresHolderTests.java
@@ -0,0 +1,276 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.repositories.s3.async;
+
+import org.opensearch.common.blobstore.stream.write.WritePriority;
+import org.opensearch.repositories.s3.GenericStatsMetricPublisher;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.mockito.Mockito;
+
+import static org.opensearch.repositories.s3.async.TransferSemaphoresHolder.TypeSemaphore.PermitType;
+
+public class TransferSemaphoresHolderTests extends OpenSearchTestCase {
+
+    public void testAllocation() {
+        int availablePermits = randomIntBetween(5, 20);
+        double priorityAllocation = randomDoubleBetween(0.1, 0.9, true);
+        int normalPermits = (int) (availablePermits * priorityAllocation);
+        int lowPermits = availablePermits - normalPermits;
+        GenericStatsMetricPublisher genericStatsPublisher = new GenericStatsMetricPublisher(10000L, 10, 10000L, 10);
+        TransferSemaphoresHolder transferSemaphoresHolder = new TransferSemaphoresHolder(
+            normalPermits,
+            lowPermits,
+            1,
+            TimeUnit.NANOSECONDS,
+            genericStatsPublisher
+        );
+        assertEquals(normalPermits, transferSemaphoresHolder.getNormalPriorityPermits());
+        assertEquals(lowPermits, transferSemaphoresHolder.getLowPriorityPermits());
+        assertEquals(0, genericStatsPublisher.getAcquiredNormalPriorityPermits());
+        assertEquals(0, genericStatsPublisher.getAcquiredLowPriorityPermits());
+    }
+
+    public void testLowPriorityEventPermitAcquisition() throws InterruptedException {
+        int availablePermits = randomIntBetween(5, 50);
+        double priorityAllocation = randomDoubleBetween(0.1, 0.9, true);
+        int normalPermits = (int) (availablePermits * priorityAllocation);
+        int lowPermits = availablePermits - normalPermits;
+        GenericStatsMetricPublisher genericStatsPublisher = new GenericStatsMetricPublisher(10000L, 10, 10000L, 10);
+        TransferSemaphoresHolder transferSemaphoresHolder = new TransferSemaphoresHolder(
+            normalPermits,
+            lowPermits,
+            1,
+            TimeUnit.NANOSECONDS,
+            genericStatsPublisher
+        );
+
+        List<TransferSemaphoresHolder.TypeSemaphore> semaphores = new ArrayList<>();
+        int normalPermitsEligibleForLowEvents = normalPermits - (int) (normalPermits * 0.4);
+
+        int lowAcquisitionsExpected = (normalPermitsEligibleForLowEvents + lowPermits);
+        for (int i = 0; i < lowAcquisitionsExpected; i++) {
+            TransferSemaphoresHolder.RequestContext requestContext = transferSemaphoresHolder.createRequestContext();
+            TransferSemaphoresHolder.TypeSemaphore acquiredSemaphore = transferSemaphoresHolder.acquirePermit(
+                WritePriority.LOW,
+                requestContext
+            );
+            semaphores.add(acquiredSemaphore);
+            if (i >= lowPermits) {
+                assertEquals(PermitType.NORMAL, acquiredSemaphore.getType());
+            } else {
+                assertEquals(PermitType.LOW, acquiredSemaphore.getType());
+            }
+        }
+
+        for (int i = 0; i < normalPermits - normalPermitsEligibleForLowEvents; i++) {
+            TransferSemaphoresHolder.RequestContext requestContext = transferSemaphoresHolder.createRequestContext();
+            TransferSemaphoresHolder.TypeSemaphore acquiredSemaphore = transferSemaphoresHolder.acquirePermit(
+                WritePriority.NORMAL,
+                requestContext
+            );
+            assertEquals(PermitType.NORMAL, acquiredSemaphore.getType());
+            semaphores.add(acquiredSemaphore);
+        }
+
+        TransferSemaphoresHolder.RequestContext requestContext = transferSemaphoresHolder.createRequestContext();
+        TransferSemaphoresHolder.TypeSemaphore acquiredSemaphore = transferSemaphoresHolder.acquirePermit(
+            WritePriority.LOW,
+            requestContext
+        );
+        assertNull(acquiredSemaphore);
+
+        assertEquals(availablePermits, semaphores.size());
+        semaphores.forEach(Semaphore::release);
+        assertEquals(normalPermits, transferSemaphoresHolder.getNormalPriorityPermits());
+        assertEquals(lowPermits, transferSemaphoresHolder.getLowPriorityPermits());
+        assertEquals(0, genericStatsPublisher.getAcquiredNormalPriorityPermits());
+        assertEquals(0, genericStatsPublisher.getAcquiredLowPriorityPermits());
+
+    }
+
+    public void testNormalPermitEventAcquisition() throws InterruptedException {
+        int availablePermits = randomIntBetween(5, 50);
+        double priorityAllocation = randomDoubleBetween(0.1, 0.9, true);
+        int normalPermits = (int) (availablePermits * priorityAllocation);
+        int lowPermits = availablePermits - normalPermits;
+        GenericStatsMetricPublisher genericStatsPublisher = new GenericStatsMetricPublisher(10000L, 10, 10000L, 10);
+        TransferSemaphoresHolder transferSemaphoresHolder = new TransferSemaphoresHolder(
+            normalPermits,
+            lowPermits,
+            1,
+            TimeUnit.NANOSECONDS,
+            genericStatsPublisher
+        );
+
+        List<TransferSemaphoresHolder.TypeSemaphore> semaphores = new ArrayList<>();
+        List<TransferSemaphoresHolder.TypeSemaphore> lowSemaphores = new ArrayList<>();
+        int normalAcquisitionsExpected = normalPermits + lowPermits;
+        TransferSemaphoresHolder.RequestContext requestContext = transferSemaphoresHolder.createRequestContext();
+        for (int i = 0; i < normalAcquisitionsExpected; i++) {
+            TransferSemaphoresHolder.TypeSemaphore acquiredSemaphore = transferSemaphoresHolder.acquirePermit(
+                WritePriority.NORMAL,
+                requestContext
+            );
+            semaphores.add(acquiredSemaphore);
+            if (i >= normalPermits) {
+                assertEquals(PermitType.LOW, acquiredSemaphore.getType());
+                lowSemaphores.add(acquiredSemaphore);
+            } else {
+                assertEquals(PermitType.NORMAL, acquiredSemaphore.getType());
+            }
+        }
+        assertEquals(availablePermits, semaphores.size());
+
+        int lowAcquired = lowPermits;
+
+        Semaphore removedLowSemaphore = lowSemaphores.remove(0);
+        removedLowSemaphore.release();
+        semaphores.remove(removedLowSemaphore);
+
+        requestContext = transferSemaphoresHolder.createRequestContext();
+        TransferSemaphoresHolder.TypeSemaphore acquiredSemaphore = transferSemaphoresHolder.acquirePermit(
+            WritePriority.LOW,
+            requestContext
+        );
+        semaphores.add(acquiredSemaphore);
+        lowSemaphores.add(acquiredSemaphore);
+        while (lowAcquired > 1) {
+            requestContext = transferSemaphoresHolder.createRequestContext();
+            acquiredSemaphore = transferSemaphoresHolder.acquirePermit(WritePriority.NORMAL, requestContext);
+            assertNull(acquiredSemaphore);
+            lowAcquired--;
+        }
+
+        semaphores.forEach(Semaphore::release);
+        assertEquals(normalPermits, transferSemaphoresHolder.getNormalPriorityPermits());
+        assertEquals(lowPermits, transferSemaphoresHolder.getLowPriorityPermits());
+        assertEquals(0, genericStatsPublisher.getAcquiredNormalPriorityPermits());
+        assertEquals(0, genericStatsPublisher.getAcquiredLowPriorityPermits());
+    }
+
+    private static class TestTransferSemaphoresHolder extends TransferSemaphoresHolder {
+        AtomicInteger normalWaitCount = new AtomicInteger();
+        AtomicInteger lowWaitCount = new AtomicInteger();
+
+        /**
+         * Constructor to create semaphores holder.
+         */
+        public TestTransferSemaphoresHolder(
+            int normalPermits,
+            int lowPermits,
+            int acquireWaitDuration,
+            TimeUnit timeUnit,
+            GenericStatsMetricPublisher genericStatsMetricPublisher
+        ) throws InterruptedException {
+            super(normalPermits, lowPermits, acquireWaitDuration, timeUnit, genericStatsMetricPublisher);
+            TypeSemaphore executingNormalSemaphore = normalPrioritySemaphore;
+            TypeSemaphore executingLowSemaphore = lowPrioritySemaphore;
+
+            this.normalPrioritySemaphore = Mockito.spy(normalPrioritySemaphore);
+            this.lowPrioritySemaphore = Mockito.spy(lowPrioritySemaphore);
+            Mockito.doAnswer(invocation -> {
+                normalWaitCount.incrementAndGet();
+                return false;
+            }).when(normalPrioritySemaphore).tryAcquire(Mockito.anyLong(), Mockito.any(TimeUnit.class));
+            Mockito.doAnswer(invocation -> executingNormalSemaphore.availablePermits()).when(normalPrioritySemaphore).availablePermits();
+            Mockito.doAnswer(invocation -> executingNormalSemaphore.tryAcquire()).when(normalPrioritySemaphore).tryAcquire();
+
+            Mockito.doAnswer(invocation -> {
+                lowWaitCount.incrementAndGet();
+                return false;
+            }).when(lowPrioritySemaphore).tryAcquire(Mockito.anyLong(), Mockito.any(TimeUnit.class));
+            Mockito.doAnswer(invocation -> executingLowSemaphore.availablePermits()).when(lowPrioritySemaphore).availablePermits();
+            Mockito.doAnswer(invocation -> executingLowSemaphore.tryAcquire()).when(lowPrioritySemaphore).tryAcquire();
+        }
+    }
+
+    public void testNormalSemaphoreAcquiredWait() throws InterruptedException {
+        int availablePermits = randomIntBetween(10, 50);
+        double priorityAllocation = randomDoubleBetween(0.1, 0.9, true);
+        int normalPermits = (int) (availablePermits * priorityAllocation);
+        GenericStatsMetricPublisher genericStatsPublisher = new GenericStatsMetricPublisher(10000L, 10, 10000L, 10);
+        TestTransferSemaphoresHolder transferSemaphoresHolder = new TestTransferSemaphoresHolder(
+            normalPermits,
+            availablePermits - normalPermits,
+            5,
+            TimeUnit.MINUTES,
+            genericStatsPublisher
+        );
+
+        TransferSemaphoresHolder.RequestContext requestContext = transferSemaphoresHolder.createRequestContext();
+        TransferSemaphoresHolder.TypeSemaphore lowSemaphore = transferSemaphoresHolder.acquirePermit(WritePriority.LOW, requestContext);
+        assertEquals(PermitType.LOW, lowSemaphore.getType());
+        for (int i = 0; i < normalPermits; i++) {
+            requestContext = transferSemaphoresHolder.createRequestContext();
+            TransferSemaphoresHolder.TypeSemaphore acquiredSemaphore = transferSemaphoresHolder.acquirePermit(
+                WritePriority.NORMAL,
+                requestContext
+            );
+            assertEquals(PermitType.NORMAL, acquiredSemaphore.getType());
+        }
+
+        TransferSemaphoresHolder.TypeSemaphore acquiredSemaphore = transferSemaphoresHolder.acquirePermit(
+            WritePriority.NORMAL,
+            requestContext
+        );
+        assertNull(acquiredSemaphore);
+        assertEquals(1, transferSemaphoresHolder.normalWaitCount.get());
+        assertEquals(0, transferSemaphoresHolder.lowWaitCount.get());
+    }
+
+    public void testLowSemaphoreAcquiredWait() throws InterruptedException {
+        int availablePermits = randomIntBetween(10, 50);
+        double priorityAllocation = randomDoubleBetween(0.1, 0.9, true);
+        int normalPermits = (int) (availablePermits * priorityAllocation);
+        int lowPermits = availablePermits - normalPermits;
+        GenericStatsMetricPublisher genericStatsPublisher = new GenericStatsMetricPublisher(10000L, 10, 10000L, 10);
+        TestTransferSemaphoresHolder transferSemaphoresHolder = new TestTransferSemaphoresHolder(
+            normalPermits,
+            lowPermits,
+            5,
+            TimeUnit.MINUTES,
+            genericStatsPublisher
+        );
+
+        TransferSemaphoresHolder.RequestContext requestContext = transferSemaphoresHolder.createRequestContext();
+        int normalPermitsEligibleForLowEvents = normalPermits - (int) (normalPermits * 0.4);
+        for (int i = 0; i < normalPermitsEligibleForLowEvents; i++) {
+            TransferSemaphoresHolder.TypeSemaphore lowSemaphore = transferSemaphoresHolder.acquirePermit(
+                WritePriority.NORMAL,
+                requestContext
+            );
+            assertEquals(PermitType.NORMAL, lowSemaphore.getType());
+        }
+
+        for (int i = 0; i < lowPermits; i++) {
+            requestContext = transferSemaphoresHolder.createRequestContext();
+            TransferSemaphoresHolder.TypeSemaphore acquiredSemaphore = transferSemaphoresHolder.acquirePermit(
+                WritePriority.LOW,
+                requestContext
+            );
+            assertEquals(PermitType.LOW, acquiredSemaphore.getType());
+        }
+
+        TransferSemaphoresHolder.TypeSemaphore acquiredSemaphore = transferSemaphoresHolder.acquirePermit(
+            WritePriority.LOW,
+            requestContext
+        );
+        assertNull(acquiredSemaphore);
+        assertEquals(1, transferSemaphoresHolder.lowWaitCount.get());
+        assertEquals(0, transferSemaphoresHolder.normalWaitCount.get());
+    }
+
+}
diff --git a/plugins/transport-nio/licenses/netty-buffer-4.1.107.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-buffer-4.1.107.Final.jar.sha1
deleted file mode 100644
index beb44fc0f4cf9..0000000000000
--- a/plugins/transport-nio/licenses/netty-buffer-4.1.107.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-8509a72b8a5a2d33d611e99254aed39765c3ad82
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-buffer-4.1.109.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-buffer-4.1.109.Final.jar.sha1
new file mode 100644
index 0000000000000..76b51cdae3867
--- /dev/null
+++ b/plugins/transport-nio/licenses/netty-buffer-4.1.109.Final.jar.sha1
@@ -0,0 +1 @@
+9d21d602ad7c639fa16b1d26559065d310a34c51
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-codec-4.1.107.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-4.1.107.Final.jar.sha1
deleted file mode 100644
index 4c74bb06fd83b..0000000000000
--- a/plugins/transport-nio/licenses/netty-codec-4.1.107.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-0a1d32debf2ed07c5852ab5b2904c43adb76c39e
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-codec-4.1.109.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-4.1.109.Final.jar.sha1
new file mode 100644
index 0000000000000..1bccee872152d
--- /dev/null
+++ b/plugins/transport-nio/licenses/netty-codec-4.1.109.Final.jar.sha1
@@ -0,0 +1 @@
+16e0b2beb49318a549d3ba5d66d707bd5daa8c97
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-codec-http-4.1.107.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-http-4.1.107.Final.jar.sha1
deleted file mode 100644
index 38eb2e5bad80a..0000000000000
--- a/plugins/transport-nio/licenses/netty-codec-http-4.1.107.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-04d8e9e51b7254bd26a42fe17bdcae32e4c6ebb3
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-codec-http-4.1.109.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-codec-http-4.1.109.Final.jar.sha1
new file mode 100644
index 0000000000000..3423fb94e8497
--- /dev/null
+++ b/plugins/transport-nio/licenses/netty-codec-http-4.1.109.Final.jar.sha1
@@ -0,0 +1 @@
+6dca43cedc0b2dc6bf57bdc85fce6ffca3e6b72a
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-common-4.1.107.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-common-4.1.107.Final.jar.sha1
deleted file mode 100644
index bbe91c6ccfb1d..0000000000000
--- a/plugins/transport-nio/licenses/netty-common-4.1.107.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-4f17a547530d64becd7179507b25f4154bcfba57
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-common-4.1.109.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-common-4.1.109.Final.jar.sha1
new file mode 100644
index 0000000000000..5172500557f8b
--- /dev/null
+++ b/plugins/transport-nio/licenses/netty-common-4.1.109.Final.jar.sha1
@@ -0,0 +1 @@
+da63e54ee1ca69abf4206cb74fadef7f50850911
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-handler-4.1.107.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-handler-4.1.107.Final.jar.sha1
deleted file mode 100644
index ba27b38632622..0000000000000
--- a/plugins/transport-nio/licenses/netty-handler-4.1.107.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d4c6b05f4d9aca117981297fb7f02953102ebb5e
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-handler-4.1.109.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-handler-4.1.109.Final.jar.sha1
new file mode 100644
index 0000000000000..cabe61b300523
--- /dev/null
+++ b/plugins/transport-nio/licenses/netty-handler-4.1.109.Final.jar.sha1
@@ -0,0 +1 @@
+9167863307b3c44cc12262e7b5512de3499b9c4a
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-resolver-4.1.107.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-resolver-4.1.107.Final.jar.sha1
deleted file mode 100644
index 3bc0f7b3fed09..0000000000000
--- a/plugins/transport-nio/licenses/netty-resolver-4.1.107.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-dfee84308341a42131dd0f8ac0e1e02d627c19f3
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-resolver-4.1.109.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-resolver-4.1.109.Final.jar.sha1
new file mode 100644
index 0000000000000..14e21cc0cdb60
--- /dev/null
+++ b/plugins/transport-nio/licenses/netty-resolver-4.1.109.Final.jar.sha1
@@ -0,0 +1 @@
+055485ac976e27c8bb67ee111a8490c58f67b70c
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-transport-4.1.107.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-transport-4.1.107.Final.jar.sha1
deleted file mode 100644
index 19419999300dd..0000000000000
--- a/plugins/transport-nio/licenses/netty-transport-4.1.107.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d6a105c621b47d1410e0e09419d7209d2d46e914
\ No newline at end of file
diff --git a/plugins/transport-nio/licenses/netty-transport-4.1.109.Final.jar.sha1 b/plugins/transport-nio/licenses/netty-transport-4.1.109.Final.jar.sha1
new file mode 100644
index 0000000000000..6b23d0883e31f
--- /dev/null
+++ b/plugins/transport-nio/licenses/netty-transport-4.1.109.Final.jar.sha1
@@ -0,0 +1 @@
+79e3b07d58ef03c7a860d48f932b720675aa8bd3
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.107.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.107.Final.jar.sha1
deleted file mode 100644
index beb44fc0f4cf9..0000000000000
--- a/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.107.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-8509a72b8a5a2d33d611e99254aed39765c3ad82
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.109.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.109.Final.jar.sha1
new file mode 100644
index 0000000000000..76b51cdae3867
--- /dev/null
+++ b/plugins/transport-reactor-netty4/licenses/netty-buffer-4.1.109.Final.jar.sha1
@@ -0,0 +1 @@
+9d21d602ad7c639fa16b1d26559065d310a34c51
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.107.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.107.Final.jar.sha1
deleted file mode 100644
index 4c74bb06fd83b..0000000000000
--- a/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.107.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-0a1d32debf2ed07c5852ab5b2904c43adb76c39e
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.109.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.109.Final.jar.sha1
new file mode 100644
index 0000000000000..1bccee872152d
--- /dev/null
+++ b/plugins/transport-reactor-netty4/licenses/netty-codec-4.1.109.Final.jar.sha1
@@ -0,0 +1 @@
+16e0b2beb49318a549d3ba5d66d707bd5daa8c97
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.107.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.107.Final.jar.sha1
deleted file mode 100644
index d823de7ffadd4..0000000000000
--- a/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.107.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-945e8ad5ab7ec4f11fb0257d2594af0cfae1d4b7
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.109.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.109.Final.jar.sha1
new file mode 100644
index 0000000000000..b13a709f1c449
--- /dev/null
+++ b/plugins/transport-reactor-netty4/licenses/netty-codec-dns-4.1.109.Final.jar.sha1
@@ -0,0 +1 @@
+ee231baee2cc9f1300ecc0d9a1e8bb9b31db02fa
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.107.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.107.Final.jar.sha1
deleted file mode 100644
index 38eb2e5bad80a..0000000000000
--- a/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.107.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-04d8e9e51b7254bd26a42fe17bdcae32e4c6ebb3
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.109.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.109.Final.jar.sha1
new file mode 100644
index 0000000000000..3423fb94e8497
--- /dev/null
+++ b/plugins/transport-reactor-netty4/licenses/netty-codec-http-4.1.109.Final.jar.sha1
@@ -0,0 +1 @@
+6dca43cedc0b2dc6bf57bdc85fce6ffca3e6b72a
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.107.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.107.Final.jar.sha1
deleted file mode 100644
index 5b3d3311edc9f..0000000000000
--- a/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.107.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-3885ffe7dd05c9773df70c61009f34a5a8a383ec
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.109.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.109.Final.jar.sha1
new file mode 100644
index 0000000000000..b83ad36222d07
--- /dev/null
+++ b/plugins/transport-reactor-netty4/licenses/netty-codec-http2-4.1.109.Final.jar.sha1
@@ -0,0 +1 @@
+6bd4a54b69a81356393f6e4621bad40754f8a5a2
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-common-4.1.107.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-common-4.1.107.Final.jar.sha1
deleted file mode 100644
index bbe91c6ccfb1d..0000000000000
--- a/plugins/transport-reactor-netty4/licenses/netty-common-4.1.107.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-4f17a547530d64becd7179507b25f4154bcfba57
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-common-4.1.109.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-common-4.1.109.Final.jar.sha1
new file mode 100644
index 0000000000000..5172500557f8b
--- /dev/null
+++ b/plugins/transport-reactor-netty4/licenses/netty-common-4.1.109.Final.jar.sha1
@@ -0,0 +1 @@
+da63e54ee1ca69abf4206cb74fadef7f50850911
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.107.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.107.Final.jar.sha1
deleted file mode 100644
index ba27b38632622..0000000000000
--- a/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.107.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d4c6b05f4d9aca117981297fb7f02953102ebb5e
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.109.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.109.Final.jar.sha1
new file mode 100644
index 0000000000000..cabe61b300523
--- /dev/null
+++ b/plugins/transport-reactor-netty4/licenses/netty-handler-4.1.109.Final.jar.sha1
@@ -0,0 +1 @@
+9167863307b3c44cc12262e7b5512de3499b9c4a
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.107.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.107.Final.jar.sha1
deleted file mode 100644
index 3bc0f7b3fed09..0000000000000
--- a/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.107.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-dfee84308341a42131dd0f8ac0e1e02d627c19f3
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.109.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.109.Final.jar.sha1
new file mode 100644
index 0000000000000..14e21cc0cdb60
--- /dev/null
+++ b/plugins/transport-reactor-netty4/licenses/netty-resolver-4.1.109.Final.jar.sha1
@@ -0,0 +1 @@
+055485ac976e27c8bb67ee111a8490c58f67b70c
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.107.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.107.Final.jar.sha1
deleted file mode 100644
index a62cb0fefcc40..0000000000000
--- a/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.107.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d655d09e972dee46f580dbcf41c0d1356aea9e1b
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.109.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.109.Final.jar.sha1
new file mode 100644
index 0000000000000..b42cdc2835eb0
--- /dev/null
+++ b/plugins/transport-reactor-netty4/licenses/netty-resolver-dns-4.1.109.Final.jar.sha1
@@ -0,0 +1 @@
+5f4d858234b557b73631a24e562bb89fc5399cad
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.107.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.107.Final.jar.sha1
deleted file mode 100644
index 19419999300dd..0000000000000
--- a/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.107.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-d6a105c621b47d1410e0e09419d7209d2d46e914
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.109.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.109.Final.jar.sha1
new file mode 100644
index 0000000000000..6b23d0883e31f
--- /dev/null
+++ b/plugins/transport-reactor-netty4/licenses/netty-transport-4.1.109.Final.jar.sha1
@@ -0,0 +1 @@
+79e3b07d58ef03c7a860d48f932b720675aa8bd3
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.107.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.107.Final.jar.sha1
deleted file mode 100644
index 407ecaffdad30..0000000000000
--- a/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.107.Final.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-4d61d4959741109b3eccd7337f11fc89fa90a74a
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.109.Final.jar.sha1 b/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.109.Final.jar.sha1
new file mode 100644
index 0000000000000..5afeb9627c9b5
--- /dev/null
+++ b/plugins/transport-reactor-netty4/licenses/netty-transport-native-unix-common-4.1.109.Final.jar.sha1
@@ -0,0 +1 @@
+da7fe1e6943cbab8ee48df2beadc2c8304f347a2
\ No newline at end of file
diff --git a/plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/ReactorNetty4HttpServerTransport.java b/plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/ReactorNetty4HttpServerTransport.java
index d4a5a9ad83af6..bd1646d753016 100644
--- a/plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/ReactorNetty4HttpServerTransport.java
+++ b/plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/ReactorNetty4HttpServerTransport.java
@@ -8,6 +8,7 @@
 
 package org.opensearch.http.reactor.netty4;
 
+import org.opensearch.common.Nullable;
 import org.opensearch.common.network.NetworkService;
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Setting;
@@ -23,21 +24,33 @@
 import org.opensearch.http.HttpChannel;
 import org.opensearch.http.HttpReadTimeoutException;
 import org.opensearch.http.HttpServerChannel;
+import org.opensearch.http.reactor.netty4.ssl.SslUtils;
+import org.opensearch.plugins.SecureHttpTransportSettingsProvider;
 import org.opensearch.telemetry.tracing.Tracer;
 import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.transport.reactor.SharedGroupFactory;
 import org.opensearch.transport.reactor.netty4.Netty4Utils;
 
+import javax.net.ssl.SSLEngine;
+import javax.net.ssl.SSLException;
+import javax.net.ssl.SSLSessionContext;
+
 import java.net.InetSocketAddress;
 import java.net.SocketOption;
 import java.time.Duration;
+import java.util.Arrays;
+import java.util.List;
 
+import io.netty.buffer.ByteBufAllocator;
 import io.netty.channel.ChannelOption;
 import io.netty.channel.socket.nio.NioChannelOption;
 import io.netty.handler.codec.http.DefaultLastHttpContent;
 import io.netty.handler.codec.http.FullHttpResponse;
 import io.netty.handler.codec.http.HttpContent;
+import io.netty.handler.ssl.ApplicationProtocolNegotiator;
+import io.netty.handler.ssl.SslContext;
 import io.netty.handler.timeout.ReadTimeoutException;
+import io.netty.util.ReferenceCountUtil;
 import org.reactivestreams.Publisher;
 import reactor.core.publisher.Mono;
 import reactor.core.scheduler.Scheduler;
@@ -116,6 +129,7 @@ public class ReactorNetty4HttpServerTransport extends AbstractHttpServerTranspor
     private final ByteSizeValue maxInitialLineLength;
     private final ByteSizeValue maxHeaderSize;
     private final ByteSizeValue maxChunkSize;
+    private final SecureHttpTransportSettingsProvider secureHttpTransportSettingsProvider;
     private volatile SharedGroupFactory.SharedGroup sharedGroup;
     private volatile DisposableServer disposableServer;
     private volatile Scheduler scheduler;
@@ -142,6 +156,45 @@ public ReactorNetty4HttpServerTransport(
         ClusterSettings clusterSettings,
         SharedGroupFactory sharedGroupFactory,
         Tracer tracer
+    ) {
+        this(
+            settings,
+            networkService,
+            bigArrays,
+            threadPool,
+            xContentRegistry,
+            dispatcher,
+            clusterSettings,
+            sharedGroupFactory,
+            null,
+            tracer
+        );
+    }
+
+    /**
+     * Creates new HTTP transport implementations based on Reactor Netty (see please {@link HttpServer}).
+     * @param settings settings
+     * @param networkService network service
+     * @param bigArrays big array allocator
+     * @param threadPool thread pool instance
+     * @param xContentRegistry XContent registry instance
+     * @param dispatcher dispatcher instance
+     * @param clusterSettings cluster settings
+     * @param sharedGroupFactory shared group factory
+     * @param secureHttpTransportSettingsProvider secure HTTP transport settings provider
+     * @param tracer tracer instance
+     */
+    public ReactorNetty4HttpServerTransport(
+        Settings settings,
+        NetworkService networkService,
+        BigArrays bigArrays,
+        ThreadPool threadPool,
+        NamedXContentRegistry xContentRegistry,
+        Dispatcher dispatcher,
+        ClusterSettings clusterSettings,
+        SharedGroupFactory sharedGroupFactory,
+        @Nullable SecureHttpTransportSettingsProvider secureHttpTransportSettingsProvider,
+        Tracer tracer
     ) {
         super(settings, networkService, bigArrays, threadPool, xContentRegistry, dispatcher, clusterSettings, tracer);
         Netty4Utils.setAvailableProcessors(OpenSearchExecutors.NODE_PROCESSORS_SETTING.get(settings));
@@ -152,6 +205,7 @@ public ReactorNetty4HttpServerTransport(
         this.maxChunkSize = SETTING_HTTP_MAX_CHUNK_SIZE.get(settings);
         this.maxHeaderSize = SETTING_HTTP_MAX_HEADER_SIZE.get(settings);
         this.maxInitialLineLength = SETTING_HTTP_MAX_INITIAL_LINE_LENGTH.get(settings);
+        this.secureHttpTransportSettingsProvider = secureHttpTransportSettingsProvider;
     }
 
     /**
@@ -160,7 +214,7 @@ public ReactorNetty4HttpServerTransport(
      */
     @Override
     protected HttpServerChannel bind(InetSocketAddress socketAddress) throws Exception {
-        final HttpServer server = configureChannelOptions(
+        final HttpServer server = configure(
             HttpServer.create()
                 .httpFormDecoder(builder -> builder.scheduler(scheduler))
                 .idleTimeout(Duration.ofMillis(connectTimeoutMillis))
@@ -173,7 +227,6 @@ protected HttpServerChannel bind(InetSocketAddress socketAddress) throws Excepti
                         .maxHeaderSize(maxHeaderSize.bytesAsInt())
                         .maxInitialLineLength(maxInitialLineLength.bytesAsInt())
                 )
-                .protocol(HttpProtocol.HTTP11, HttpProtocol.H2C)
                 .handle((req, res) -> incomingRequest(req, res))
         );
 
@@ -181,8 +234,8 @@ protected HttpServerChannel bind(InetSocketAddress socketAddress) throws Excepti
         return new ReactorNetty4HttpServerChannel(disposableServer.channel());
     }
 
-    private HttpServer configureChannelOptions(final HttpServer server1) {
-        HttpServer configured = server1.childOption(ChannelOption.TCP_NODELAY, SETTING_HTTP_TCP_NO_DELAY.get(settings))
+    private HttpServer configure(final HttpServer server) throws Exception {
+        HttpServer configured = server.childOption(ChannelOption.TCP_NODELAY, SETTING_HTTP_TCP_NO_DELAY.get(settings))
             .childOption(ChannelOption.SO_KEEPALIVE, SETTING_HTTP_TCP_KEEP_ALIVE.get(settings));
 
         if (SETTING_HTTP_TCP_KEEP_ALIVE.get(settings)) {
@@ -229,6 +282,65 @@ private HttpServer configureChannelOptions(final HttpServer server1) {
         configured = configured.option(ChannelOption.SO_REUSEADDR, reuseAddress);
         configured = configured.childOption(ChannelOption.SO_REUSEADDR, reuseAddress);
 
+        // Configure SSL context if available
+        if (secureHttpTransportSettingsProvider != null) {
+            final SSLEngine engine = secureHttpTransportSettingsProvider.buildSecureHttpServerEngine(settings, this)
+                .orElseGet(SslUtils::createDefaultServerSSLEngine);
+
+            try {
+                final List<String> cipherSuites = Arrays.asList(engine.getEnabledCipherSuites());
+                final List<String> applicationProtocols = Arrays.asList(engine.getSSLParameters().getApplicationProtocols());
+
+                configured = configured.secure(spec -> spec.sslContext(new SslContext() {
+                    @Override
+                    public SSLSessionContext sessionContext() {
+                        throw new UnsupportedOperationException(); /* server only, should never be called */
+                    }
+
+                    @Override
+                    public SSLEngine newEngine(ByteBufAllocator alloc, String peerHost, int peerPort) {
+                        throw new UnsupportedOperationException(); /* server only, should never be called */
+                    }
+
+                    @Override
+                    public SSLEngine newEngine(ByteBufAllocator alloc) {
+                        try {
+                            return secureHttpTransportSettingsProvider.buildSecureHttpServerEngine(
+                                settings,
+                                ReactorNetty4HttpServerTransport.this
+                            ).orElseGet(SslUtils::createDefaultServerSSLEngine);
+                        } catch (final SSLException ex) {
+                            throw new UnsupportedOperationException("Unable to create SSLEngine", ex);
+                        }
+                    }
+
+                    @Override
+                    public boolean isClient() {
+                        return false; /* server only */
+                    }
+
+                    @Override
+                    public List<String> cipherSuites() {
+                        return cipherSuites;
+                    }
+
+                    @Override
+                    public ApplicationProtocolNegotiator applicationProtocolNegotiator() {
+                        return new ApplicationProtocolNegotiator() {
+                            @Override
+                            public List<String> protocols() {
+                                return applicationProtocols;
+                            }
+                        };
+                    }
+                }).build()).protocol(HttpProtocol.HTTP11, HttpProtocol.H2);
+            } finally {
+                ReferenceCountUtil.release(engine);
+            }
+        } else {
+            configured = configured.protocol(HttpProtocol.HTTP11, HttpProtocol.H2C);
+        }
+
         return configured;
     }
 
@@ -302,6 +414,11 @@ protected void doStart() {
         }
     }
 
+    /**
+     * Exception handler
+     * @param channel HTTP channel
+     * @param cause exception occurred
+     */
     @Override
     public void onException(HttpChannel channel, Exception cause) {
         if (cause instanceof ReadTimeoutException) {
diff --git a/plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/ssl/SslUtils.java b/plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/ssl/SslUtils.java
new file mode 100644
index 0000000000000..a0b4319c7c2d1
--- /dev/null
+++ b/plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/ssl/SslUtils.java
@@ -0,0 +1,42 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+package org.opensearch.http.reactor.netty4.ssl;
+
+import org.opensearch.OpenSearchSecurityException;
+
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.SSLEngine;
+
+import java.security.NoSuchAlgorithmException;
+
+/**
+ * Helper class for creating default SSL engines
+ */
+public class SslUtils {
+    private static final String[] DEFAULT_SSL_PROTOCOLS = { "TLSv1.3", "TLSv1.2", "TLSv1.1" };
+
+    private SslUtils() {}
+
+    /**
+     * Creates default server {@link SSLEngine} instance
+     * @return default server {@link SSLEngine} instance
+     */
+    public static SSLEngine createDefaultServerSSLEngine() {
+        try {
+            final SSLEngine engine = SSLContext.getDefault().createSSLEngine();
+            engine.setEnabledProtocols(DEFAULT_SSL_PROTOCOLS);
+            engine.setUseClientMode(false);
+            return engine;
+        } catch (final NoSuchAlgorithmException ex) {
+            throw new OpenSearchSecurityException("Unable to initialize default server SSL engine", ex);
+        }
+    }
+}
diff --git a/plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/ssl/package-info.java b/plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/ssl/package-info.java
new file mode 100644
index 0000000000000..95dbd2d6bd9ca
--- /dev/null
+++ b/plugins/transport-reactor-netty4/src/main/java/org/opensearch/http/reactor/netty4/ssl/package-info.java
@@ -0,0 +1,12 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/**
+ * SSL supporting utility classes
+ */
+package org.opensearch.http.reactor.netty4.ssl;
diff --git a/plugins/transport-reactor-netty4/src/main/java/org/opensearch/transport/reactor/ReactorNetty4Plugin.java b/plugins/transport-reactor-netty4/src/main/java/org/opensearch/transport/reactor/ReactorNetty4Plugin.java
index dc310c3793109..6e5b0215b58a4 100644
--- a/plugins/transport-reactor-netty4/src/main/java/org/opensearch/transport/reactor/ReactorNetty4Plugin.java
+++ b/plugins/transport-reactor-netty4/src/main/java/org/opensearch/transport/reactor/ReactorNetty4Plugin.java
@@ -17,9 +17,11 @@
 import org.opensearch.core.indices.breaker.CircuitBreakerService;
 import org.opensearch.core.xcontent.NamedXContentRegistry;
 import org.opensearch.http.HttpServerTransport;
+import org.opensearch.http.HttpServerTransport.Dispatcher;
 import org.opensearch.http.reactor.netty4.ReactorNetty4HttpServerTransport;
 import org.opensearch.plugins.NetworkPlugin;
 import org.opensearch.plugins.Plugin;
+import org.opensearch.plugins.SecureHttpTransportSettingsProvider;
 import org.opensearch.telemetry.tracing.Tracer;
 import org.opensearch.threadpool.ThreadPool;
 
@@ -38,6 +40,11 @@ public class ReactorNetty4Plugin extends Plugin implements NetworkPlugin {
      */
     public static final String REACTOR_NETTY_HTTP_TRANSPORT_NAME = "reactor-netty4";
 
+    /**
+     * The name of new experimental secure HTTP transport implementations based on Reactor Netty.
+     */
+    public static final String REACTOR_NETTY_SECURE_HTTP_TRANSPORT_NAME = "reactor-netty4-secure";
+
     private final SetOnce<SharedGroupFactory> groupFactory = new SetOnce<>();
 
     /**
@@ -91,6 +98,53 @@ public Map<String, Supplier<HttpServerTransport>> getHttpTransports(
                 dispatcher,
                 clusterSettings,
                 getSharedGroupFactory(settings),
+                null, /* no security settings provider */
+                tracer
+            )
+        );
+    }
+
+    /**
+     * Returns a map of {@link HttpServerTransport} suppliers.
+     * See {@link org.opensearch.common.network.NetworkModule#HTTP_TYPE_SETTING} to configure a specific implementation.
+     * @param settings settings
+     * @param networkService network service
+     * @param bigArrays big array allocator
+     * @param pageCacheRecycler page cache recycler instance
+     * @param circuitBreakerService circuit breaker service instance
+     * @param threadPool thread pool instance
+     * @param xContentRegistry XContent registry instance
+     * @param dispatcher dispatcher instance
+     * @param clusterSettings cluster settings
+     * @param secureHttpTransportSettingsProvider secure HTTP transport settings provider
+     * @param tracer tracer instance
+     */
+    @Override
+    public Map<String, Supplier<HttpServerTransport>> getSecureHttpTransports(
+        Settings settings,
+        ThreadPool threadPool,
+        BigArrays bigArrays,
+        PageCacheRecycler pageCacheRecycler,
+        CircuitBreakerService circuitBreakerService,
+        NamedXContentRegistry xContentRegistry,
+        NetworkService networkService,
+        Dispatcher dispatcher,
+        ClusterSettings clusterSettings,
+        SecureHttpTransportSettingsProvider secureHttpTransportSettingsProvider,
+        Tracer tracer
+    ) {
+        return Collections.singletonMap(
+            REACTOR_NETTY_SECURE_HTTP_TRANSPORT_NAME,
+            () -> new ReactorNetty4HttpServerTransport(
+                settings,
+                networkService,
+                bigArrays,
+                threadPool,
+                xContentRegistry,
+                dispatcher,
+                clusterSettings,
+                getSharedGroupFactory(settings),
+                secureHttpTransportSettingsProvider,
                 tracer
             )
         );
diff --git a/plugins/transport-reactor-netty4/src/test/java/org/opensearch/http/reactor/netty4/ReactorHttpClient.java b/plugins/transport-reactor-netty4/src/test/java/org/opensearch/http/reactor/netty4/ReactorHttpClient.java
index 443ecd0f40ead..920c895205023 100644
--- a/plugins/transport-reactor-netty4/src/test/java/org/opensearch/http/reactor/netty4/ReactorHttpClient.java
+++ b/plugins/transport-reactor-netty4/src/test/java/org/opensearch/http/reactor/netty4/ReactorHttpClient.java
@@ -15,6 +15,7 @@
 
 import org.opensearch.common.collect.Tuple;
 import org.opensearch.tasks.Task;
+import org.opensearch.test.OpenSearchTestCase;
 
 import java.io.Closeable;
 import java.net.InetSocketAddress;
@@ -38,10 +39,14 @@
 import io.netty.handler.codec.http.HttpResponse;
 import io.netty.handler.codec.http.HttpVersion;
 import io.netty.handler.codec.http2.HttpConversionUtil;
+import io.netty.handler.ssl.ClientAuth;
+import io.netty.handler.ssl.util.InsecureTrustManagerFactory;
 import io.netty.resolver.DefaultAddressResolverGroup;
 import reactor.core.publisher.Flux;
 import reactor.core.publisher.Mono;
 import reactor.core.publisher.ParallelFlux;
+import reactor.netty.http.Http11SslContextSpec;
+import reactor.netty.http.Http2SslContextSpec;
 import reactor.netty.http.client.HttpClient;
 
 import static io.netty.handler.codec.http.HttpHeaderNames.HOST;
@@ -50,8 +55,9 @@
 /**
  * Tiny helper to send http requests over netty.
  */
-class ReactorHttpClient implements Closeable {
+public class ReactorHttpClient implements Closeable {
     private final boolean compression;
+    private final boolean secure;
 
     static Collection<String> returnHttpResponseBodies(Collection<FullHttpResponse> responses) {
         List<String> list = new ArrayList<>(responses.size());
@@ -69,16 +75,21 @@ static Collection<String> returnOpaqueIds(Collection<FullHttpResponse> responses
         return list;
     }
 
-    ReactorHttpClient(boolean compression) {
+    public ReactorHttpClient(boolean compression, boolean secure) {
         this.compression = compression;
+        this.secure = secure;
     }
 
-    static ReactorHttpClient create() {
+    public static ReactorHttpClient create() {
         return create(true);
     }
 
-    static ReactorHttpClient create(boolean compression) {
-        return new ReactorHttpClient(compression);
+    public static ReactorHttpClient create(boolean compression) {
+        return new ReactorHttpClient(compression, false);
+    }
+
+    public static ReactorHttpClient https() {
+        return new ReactorHttpClient(true, true);
     }
 
     public List<FullHttpResponse> get(InetSocketAddress remoteAddress, String... uris) throws InterruptedException {
@@ -92,7 +103,7 @@ public List<FullHttpResponse> get(InetSocketAddress remoteAddress, boolean order
             final FullHttpRequest httpRequest = new DefaultFullHttpRequest(HTTP_1_1, HttpMethod.GET, uris[i]);
             httpRequest.headers().add(HOST, "localhost");
             httpRequest.headers().add("X-Opaque-ID", String.valueOf(i));
-            httpRequest.headers().add(HttpConversionUtil.ExtensionHeaderNames.SCHEME.text(), "http");
+            httpRequest.headers().add(HttpConversionUtil.ExtensionHeaderNames.SCHEME.text(), secure ? "https" : "http");
             requests.add(httpRequest);
         }
 
@@ -148,7 +159,7 @@ private List<FullHttpResponse> processRequestsWithBody(
             request.headers().add(HttpHeaderNames.HOST, "localhost");
             request.headers().add(HttpHeaderNames.CONTENT_LENGTH, content.readableBytes());
             request.headers().add(HttpHeaderNames.CONTENT_TYPE, "application/json");
-            request.headers().add(HttpConversionUtil.ExtensionHeaderNames.SCHEME.text(), "http");
+            request.headers().add(HttpConversionUtil.ExtensionHeaderNames.SCHEME.text(), secure ? "https" : "http");
             request.headers().add("X-Opaque-ID", String.valueOf(i));
             requests.add(request);
         }
@@ -162,12 +173,7 @@ private List<FullHttpResponse> sendRequests(
     ) {
         final NioEventLoopGroup eventLoopGroup = new NioEventLoopGroup(1);
         try {
-            final HttpClient client = HttpClient.newConnection()
-                .resolver(DefaultAddressResolverGroup.INSTANCE)
-                .runOn(eventLoopGroup)
-                .host(remoteAddress.getHostString())
-                .port(remoteAddress.getPort())
-                .compress(compression);
+            final HttpClient client = createClient(remoteAddress, eventLoopGroup);
 
             @SuppressWarnings("unchecked")
             final Mono<FullHttpResponse>[] monos = requests.stream()
@@ -201,6 +207,29 @@ private List<FullHttpResponse> sendRequests(
         }
     }
 
+    private HttpClient createClient(final InetSocketAddress remoteAddress, final NioEventLoopGroup eventLoopGroup) {
+        final HttpClient client = HttpClient.newConnection()
+            .resolver(DefaultAddressResolverGroup.INSTANCE)
+            .runOn(eventLoopGroup)
+            .host(remoteAddress.getHostString())
+            .port(remoteAddress.getPort())
+            .compress(compression);
+
+        if (secure) {
+            return client.secure(
+                spec -> spec.sslContext(
+                    OpenSearchTestCase.randomBoolean()
+                        /* switch between HTTP 1.1/HTTP 2 randomly, both are supported */ ? Http11SslContextSpec.forClient()
+                            .configure(s -> s.clientAuth(ClientAuth.NONE).trustManager(InsecureTrustManagerFactory.INSTANCE))
+                        : Http2SslContextSpec.forClient()
+                            .configure(s -> s.clientAuth(ClientAuth.NONE).trustManager(InsecureTrustManagerFactory.INSTANCE))
+                )
+            );
+        }
+
+        return client;
+    }
+
     @Override
     public void close() {
 
diff --git a/plugins/transport-reactor-netty4/src/test/java/org/opensearch/http/reactor/netty4/ssl/SecureReactorNetty4HttpServerTransportTests.java b/plugins/transport-reactor-netty4/src/test/java/org/opensearch/http/reactor/netty4/ssl/SecureReactorNetty4HttpServerTransportTests.java
new file mode 100644
index 0000000000000..ac7687d551766
--- /dev/null
+++ b/plugins/transport-reactor-netty4/src/test/java/org/opensearch/http/reactor/netty4/ssl/SecureReactorNetty4HttpServerTransportTests.java
@@ -0,0 +1,595 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.http.reactor.netty4.ssl;
+
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.opensearch.common.network.NetworkAddress;
+import org.opensearch.common.network.NetworkService;
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Setting;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.TimeValue;
+import org.opensearch.common.util.MockBigArrays;
+import org.opensearch.common.util.MockPageCacheRecycler;
+import org.opensearch.common.util.concurrent.ThreadContext;
+import org.opensearch.core.common.bytes.BytesArray;
+import org.opensearch.core.common.transport.TransportAddress;
+import org.opensearch.core.common.unit.ByteSizeValue;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
+import org.opensearch.http.BindHttpException;
+import org.opensearch.http.CorsHandler;
+import org.opensearch.http.HttpServerTransport;
+import org.opensearch.http.HttpTransportSettings;
+import org.opensearch.http.NullDispatcher;
+import org.opensearch.http.reactor.netty4.ReactorHttpClient;
+import org.opensearch.http.reactor.netty4.ReactorNetty4HttpServerTransport;
+import org.opensearch.plugins.SecureHttpTransportSettingsProvider;
+import org.opensearch.plugins.TransportExceptionHandler;
+import org.opensearch.rest.BytesRestResponse;
+import org.opensearch.rest.RestChannel;
+import org.opensearch.rest.RestRequest;
+import org.opensearch.telemetry.tracing.noop.NoopTracer;
+import org.opensearch.test.OpenSearchTestCase;
+import org.opensearch.test.rest.FakeRestRequest;
+import org.opensearch.threadpool.TestThreadPool;
+import org.opensearch.threadpool.ThreadPool;
+import org.opensearch.transport.NettyAllocator;
+import org.opensearch.transport.reactor.SharedGroupFactory;
+import org.junit.After;
+import org.junit.Before;
+
+import javax.net.ssl.SSLEngine;
+import javax.net.ssl.SSLException;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import java.util.Optional;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import io.netty.bootstrap.Bootstrap;
+import io.netty.buffer.ByteBufAllocator;
+import io.netty.buffer.ByteBufUtil;
+import io.netty.buffer.PoolArenaMetric;
+import io.netty.buffer.PooledByteBufAllocator;
+import io.netty.buffer.PooledByteBufAllocatorMetric;
+import io.netty.buffer.Unpooled;
+import io.netty.channel.ChannelFuture;
+import io.netty.channel.ChannelHandlerAdapter;
+import io.netty.channel.ChannelInitializer;
+import io.netty.channel.ChannelOption;
+import io.netty.channel.nio.NioEventLoopGroup;
+import io.netty.channel.socket.SocketChannel;
+import io.netty.channel.socket.nio.NioSocketChannel;
+import io.netty.handler.codec.http.DefaultFullHttpRequest;
+import io.netty.handler.codec.http.DefaultHttpContent;
+import io.netty.handler.codec.http.FullHttpRequest;
+import io.netty.handler.codec.http.FullHttpResponse;
+import io.netty.handler.codec.http.HttpContent;
+import io.netty.handler.codec.http.HttpHeaderNames;
+import io.netty.handler.codec.http.HttpHeaderValues;
+import io.netty.handler.codec.http.HttpMethod;
+import io.netty.handler.codec.http.HttpResponseStatus;
+import io.netty.handler.codec.http.HttpUtil;
+import io.netty.handler.codec.http.HttpVersion;
+import io.netty.handler.ssl.SslContextBuilder;
+import io.netty.handler.ssl.util.InsecureTrustManagerFactory;
+
+import static org.opensearch.core.rest.RestStatus.OK;
+import static org.opensearch.http.HttpTransportSettings.SETTING_CORS_ALLOW_ORIGIN;
+import static org.opensearch.http.HttpTransportSettings.SETTING_CORS_ENABLED;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.is;
+
+/**
+ * Tests for the secure {@link ReactorNetty4HttpServerTransport} class.
+ */
+public class SecureReactorNetty4HttpServerTransportTests extends OpenSearchTestCase {
+
+    private NetworkService networkService;
+    private ThreadPool threadPool;
+    private MockBigArrays bigArrays;
+    private ClusterSettings clusterSettings;
+    private SecureHttpTransportSettingsProvider secureHttpTransportSettingsProvider;
+
+    @Before
+    public void setup() throws Exception {
+        networkService = new NetworkService(Collections.emptyList());
+        threadPool = new TestThreadPool("test");
+        bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService());
+        clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+
+        secureHttpTransportSettingsProvider = new SecureHttpTransportSettingsProvider() {
+            @Override
+            public Optional<TransportExceptionHandler> buildHttpServerExceptionHandler(Settings settings, HttpServerTransport transport) {
+                return Optional.empty();
+            }
+
+            @Override
+            public Optional<SSLEngine> buildSecureHttpServerEngine(Settings settings, HttpServerTransport transport) throws SSLException {
+                try {
+                    SSLEngine engine = SslContextBuilder.forServer(
+                        SecureReactorNetty4HttpServerTransportTests.class.getResourceAsStream("/certificate.crt"),
+                        SecureReactorNetty4HttpServerTransportTests.class.getResourceAsStream("/certificate.key")
+                    ).trustManager(InsecureTrustManagerFactory.INSTANCE).build().newEngine(NettyAllocator.getAllocator());
+                    return Optional.of(engine);
+                } catch (final IOException ex) {
+                    throw new SSLException(ex);
+                }
+            }
+        };
+    }
+
+    @After
+    public void shutdown() throws Exception {
+        if (threadPool != null) {
+            threadPool.shutdownNow();
+        }
+        threadPool = null;
+        networkService = null;
+        bigArrays = null;
+        clusterSettings = null;
+    }
+
+    /**
+     * Test that {@link ReactorNetty4HttpServerTransport} supports the "Expect: 100-continue" HTTP header
+     * @throws InterruptedException if the client communication with the server is interrupted
+     */
+    public void testExpectContinueHeader() throws InterruptedException {
+        final Settings settings = createSettings();
+        final int contentLength = randomIntBetween(1, HttpTransportSettings.SETTING_HTTP_MAX_CONTENT_LENGTH.get(settings).bytesAsInt());
+        runExpectHeaderTest(settings, HttpHeaderValues.CONTINUE.toString(), contentLength, HttpResponseStatus.CONTINUE);
+    }
+
+    /**
+     * Test that {@link ReactorNetty4HttpServerTransport} responds to a
+     * 100-continue expectation with too large a content-length
+     * with a 413 status.
+     * @throws InterruptedException if the client communication with the server is interrupted
+     */
+    public void testExpectContinueHeaderContentLengthTooLong() throws InterruptedException {
+        final String key = HttpTransportSettings.SETTING_HTTP_MAX_CONTENT_LENGTH.getKey();
+        final int maxContentLength = randomIntBetween(1, 104857600);
+        final Settings settings = createBuilderWithPort().put(key, maxContentLength + "b").build();
+        final int contentLength = randomIntBetween(maxContentLength + 1, Integer.MAX_VALUE);
+        runExpectHeaderTest(settings, HttpHeaderValues.CONTINUE.toString(), contentLength, HttpResponseStatus.REQUEST_ENTITY_TOO_LARGE);
+    }
+
+    /**
+     * Test that {@link ReactorNetty4HttpServerTransport} responds to an unsupported expectation with a 417 status.
+     * @throws InterruptedException if the client communication with the server is interrupted
+     */
+    public void testExpectUnsupportedExpectation() throws InterruptedException {
+        Settings settings = createSettings();
+        runExpectHeaderTest(settings, "chocolate=yummy", 0, HttpResponseStatus.EXPECTATION_FAILED);
+    }
+
+    private void runExpectHeaderTest(
+        final Settings settings,
+        final String expectation,
+        final int contentLength,
+        final HttpResponseStatus expectedStatus
+    ) throws InterruptedException {
+        final HttpServerTransport.Dispatcher dispatcher = new HttpServerTransport.Dispatcher() {
+            @Override
+            public void dispatchRequest(RestRequest request, RestChannel channel, ThreadContext threadContext) {
+                channel.sendResponse(new BytesRestResponse(OK, BytesRestResponse.TEXT_CONTENT_TYPE, new BytesArray("done")));
+            }
+
+            @Override
+            public void dispatchBadRequest(RestChannel channel, ThreadContext threadContext, Throwable cause) {
+                logger.error(
+                    new ParameterizedMessage("--> Unexpected bad request [{}]", FakeRestRequest.requestToString(channel.request())),
+                    cause
+                );
+                throw new AssertionError();
+            }
+        };
+        try (
+            ReactorNetty4HttpServerTransport transport = new ReactorNetty4HttpServerTransport(
+                settings,
+                networkService,
+                bigArrays,
+                threadPool,
+                xContentRegistry(),
+                dispatcher,
+                clusterSettings,
+                new SharedGroupFactory(settings),
+                secureHttpTransportSettingsProvider,
+                NoopTracer.INSTANCE
+            )
+        ) {
+            transport.start();
+            final TransportAddress remoteAddress = randomFrom(transport.boundAddress().boundAddresses());
+            try (ReactorHttpClient client = ReactorHttpClient.https()) {
+                final FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.POST, "/");
+                request.headers().set(HttpHeaderNames.EXPECT, expectation);
+                HttpUtil.setContentLength(request, contentLength);
+
+                // Reactor Netty 4 does not expose 100 CONTINUE response but instead just asks for content
+                final HttpContent continuationRequest = new DefaultHttpContent(Unpooled.EMPTY_BUFFER);
+                final FullHttpResponse continuationResponse = client.send(remoteAddress.address(), request, continuationRequest);
+                try {
+                    assertThat(continuationResponse.status(), is(HttpResponseStatus.OK));
+                    assertThat(new String(ByteBufUtil.getBytes(continuationResponse.content()), StandardCharsets.UTF_8), is("done"));
+                } finally {
+                    continuationResponse.release();
+                }
+            }
+        }
+    }
+
+    public void testBindUnavailableAddress() {
+        Settings initialSettings = createSettings();
+        try (
+            ReactorNetty4HttpServerTransport transport = new ReactorNetty4HttpServerTransport(
+                initialSettings,
+                networkService,
+                bigArrays,
+                threadPool,
+                xContentRegistry(),
+                new NullDispatcher(),
+                clusterSettings,
+                new SharedGroupFactory(Settings.EMPTY),
+                secureHttpTransportSettingsProvider,
+                NoopTracer.INSTANCE
+            )
+        ) {
+            transport.start();
+            TransportAddress remoteAddress = randomFrom(transport.boundAddress().boundAddresses());
+            Settings settings = Settings.builder()
+                .put("http.port", remoteAddress.getPort())
+                .put("network.host", remoteAddress.getAddress())
+                .build();
+            try (
+                ReactorNetty4HttpServerTransport otherTransport = new ReactorNetty4HttpServerTransport(
+                    settings,
+                    networkService,
+                    bigArrays,
+                    threadPool,
+                    xContentRegistry(),
+                    new NullDispatcher(),
+                    clusterSettings,
+                    new SharedGroupFactory(settings),
+                    secureHttpTransportSettingsProvider,
+                    NoopTracer.INSTANCE
+                )
+            ) {
+                BindHttpException bindHttpException = expectThrows(BindHttpException.class, otherTransport::start);
+                assertEquals("Failed to bind to " + NetworkAddress.format(remoteAddress.address()), bindHttpException.getMessage());
+            }
+        }
+    }
+
+    public void testBadRequest() throws InterruptedException {
+        final HttpServerTransport.Dispatcher dispatcher = new HttpServerTransport.Dispatcher() {
+            @Override
+            public void dispatchRequest(final RestRequest request, final RestChannel channel, final ThreadContext threadContext) {
+                logger.error("--> Unexpected successful request [{}]", FakeRestRequest.requestToString(request));
+                throw new AssertionError();
+            }
+
+            @Override
+            public void dispatchBadRequest(final RestChannel channel, final ThreadContext threadContext, final Throwable cause) {
+                logger.error("--> Unexpected bad request request");
+                throw new AssertionError(cause);
+            }
+        };
+
+        final Settings settings;
+        final int maxInitialLineLength;
+        final Setting<ByteSizeValue> httpMaxInitialLineLengthSetting = HttpTransportSettings.SETTING_HTTP_MAX_INITIAL_LINE_LENGTH;
+        if (randomBoolean()) {
+            maxInitialLineLength = httpMaxInitialLineLengthSetting.getDefault(Settings.EMPTY).bytesAsInt();
+            settings = createSettings();
+        } else {
+            maxInitialLineLength = randomIntBetween(1, 8192);
+            settings = createBuilderWithPort().put(httpMaxInitialLineLengthSetting.getKey(), maxInitialLineLength + "b").build();
+        }
+
+        try (
+            ReactorNetty4HttpServerTransport transport = new ReactorNetty4HttpServerTransport(
+                settings,
+                networkService,
+                bigArrays,
+                threadPool,
+                xContentRegistry(),
+                dispatcher,
+                clusterSettings,
+                new SharedGroupFactory(settings),
+                secureHttpTransportSettingsProvider,
+                NoopTracer.INSTANCE
+            )
+        ) {
+            transport.start();
+            final TransportAddress remoteAddress = randomFrom(transport.boundAddress().boundAddresses());
+
+            try (ReactorHttpClient client = ReactorHttpClient.https()) {
+                final String url = "/" + randomAlphaOfLength(maxInitialLineLength);
+                final FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, url);
+
+                final FullHttpResponse response = client.send(remoteAddress.address(), request);
+                try {
+                    assertThat(response.status(), equalTo(HttpResponseStatus.REQUEST_URI_TOO_LONG));
+                    assertThat(response.content().array().length, equalTo(0));
+                } finally {
+                    response.release();
+                }
+            }
+        }
+    }
+
+    public void testDispatchFailed() throws InterruptedException {
+        final HttpServerTransport.Dispatcher dispatcher = new HttpServerTransport.Dispatcher() {
+            @Override
+            public void dispatchRequest(final RestRequest request, final RestChannel channel, final ThreadContext threadContext) {
+                throw new RuntimeException("Bad things happen");
+            }
+
+            @Override
+            public void dispatchBadRequest(final RestChannel channel, final ThreadContext threadContext, final Throwable cause) {
+                logger.error("--> Unexpected bad request request");
+                throw new AssertionError(cause);
+            }
+        };
+
+        final Settings settings = createSettings();
+        try (
+            ReactorNetty4HttpServerTransport transport = new ReactorNetty4HttpServerTransport(
+                settings,
+                networkService,
+                bigArrays,
+                threadPool,
+                xContentRegistry(),
+                dispatcher,
+                clusterSettings,
+                new SharedGroupFactory(settings),
+                secureHttpTransportSettingsProvider,
+                NoopTracer.INSTANCE
+            )
+        ) {
+            transport.start();
+            final TransportAddress remoteAddress = randomFrom(transport.boundAddress().boundAddresses());
+
+            try (ReactorHttpClient client = ReactorHttpClient.https()) {
+                final FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/");
+
+                final FullHttpResponse response = client.send(remoteAddress.address(), request);
+                try {
+                    assertThat(response.status(), equalTo(HttpResponseStatus.INTERNAL_SERVER_ERROR));
+                    assertThat(response.content().array().length, equalTo(0));
+                } finally {
+                    response.release();
+                }
+            }
+        }
+    }
+
+    public void testLargeCompressedResponse() throws InterruptedException {
+        final String responseString = randomAlphaOfLength(4 * 1024 * 1024);
+        final String url = "/thing/";
+        final HttpServerTransport.Dispatcher dispatcher = new HttpServerTransport.Dispatcher() {
+
+            @Override
+            public void dispatchRequest(final RestRequest request, final RestChannel channel, final ThreadContext threadContext) {
+                if (url.equals(request.uri())) {
+                    channel.sendResponse(new BytesRestResponse(OK, responseString));
+                } else {
+                    logger.error("--> Unexpected successful uri [{}]", request.uri());
+                    throw new AssertionError();
+                }
+            }
+
+            @Override
+            public void dispatchBadRequest(final RestChannel channel, final ThreadContext threadContext, final Throwable cause) {
+                logger.error(
+                    new ParameterizedMessage("--> Unexpected bad request [{}]", FakeRestRequest.requestToString(channel.request())),
+                    cause
+                );
+                throw new AssertionError();
+            }
+
+        };
+
+        try (
+            ReactorNetty4HttpServerTransport transport = new ReactorNetty4HttpServerTransport(
+                Settings.EMPTY,
+                networkService,
+                bigArrays,
+                threadPool,
+                xContentRegistry(),
+                dispatcher,
+                clusterSettings,
+                new SharedGroupFactory(Settings.EMPTY),
+                secureHttpTransportSettingsProvider,
+                NoopTracer.INSTANCE
+            )
+        ) {
+            transport.start();
+            final TransportAddress remoteAddress = randomFrom(transport.boundAddress().boundAddresses());
+
+            try (ReactorHttpClient client = ReactorHttpClient.https()) {
+                DefaultFullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, url);
+                request.headers().add(HttpHeaderNames.ACCEPT_ENCODING, randomFrom("deflate", "gzip"));
+                long numOfHugeAllocations = getHugeAllocationCount();
+                final FullHttpResponse response = client.send(remoteAddress.address(), request);
+                try {
+                    assertThat(getHugeAllocationCount(), equalTo(numOfHugeAllocations));
+                    assertThat(response.status(), equalTo(HttpResponseStatus.OK));
+                    byte[] bytes = new byte[response.content().readableBytes()];
+                    response.content().readBytes(bytes);
+                    assertThat(new String(bytes, StandardCharsets.UTF_8), equalTo(responseString));
+                } finally {
+                    response.release();
+                }
+            }
+        }
+    }
+
+    private long getHugeAllocationCount() {
+        long numOfHugAllocations = 0;
+        ByteBufAllocator allocator = NettyAllocator.getAllocator();
+        assert allocator instanceof NettyAllocator.NoDirectBuffers;
+        ByteBufAllocator delegate = ((NettyAllocator.NoDirectBuffers) allocator).getDelegate();
+        if (delegate instanceof PooledByteBufAllocator) {
+            PooledByteBufAllocatorMetric metric = ((PooledByteBufAllocator) delegate).metric();
+            numOfHugAllocations = metric.heapArenas().stream().mapToLong(PoolArenaMetric::numHugeAllocations).sum();
+        }
+        return numOfHugAllocations;
+    }
+
+    public void testCorsRequest() throws InterruptedException {
+        final HttpServerTransport.Dispatcher dispatcher = new HttpServerTransport.Dispatcher() {
+
+            @Override
+            public void dispatchRequest(final RestRequest request, final RestChannel channel, final ThreadContext threadContext) {
+                logger.error("--> Unexpected successful request [{}]", FakeRestRequest.requestToString(request));
+                throw new AssertionError();
+            }
+
+            @Override
+            public void dispatchBadRequest(final RestChannel channel, final ThreadContext threadContext, final Throwable cause) {
+                logger.error(
+                    new ParameterizedMessage("--> Unexpected bad request [{}]", FakeRestRequest.requestToString(channel.request())),
+                    cause
+                );
+                throw new AssertionError();
+            }
+
+        };
+
+        final Settings settings = createBuilderWithPort().put(SETTING_CORS_ENABLED.getKey(), true)
+            .put(SETTING_CORS_ALLOW_ORIGIN.getKey(), "test-cors.org")
+            .build();
+
+        try (
+            ReactorNetty4HttpServerTransport transport = new ReactorNetty4HttpServerTransport(
+                settings,
+                networkService,
+                bigArrays,
+                threadPool,
+                xContentRegistry(),
+                dispatcher,
+                new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
+                new SharedGroupFactory(settings),
+                secureHttpTransportSettingsProvider,
+                NoopTracer.INSTANCE
+            )
+        ) {
+            transport.start();
+            final TransportAddress remoteAddress = randomFrom(transport.boundAddress().boundAddresses());
+
+            // Test pre-flight request
+            try (ReactorHttpClient client = ReactorHttpClient.https()) {
+                final FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.OPTIONS, "/");
+                request.headers().add(CorsHandler.ORIGIN, "test-cors.org");
+                request.headers().add(CorsHandler.ACCESS_CONTROL_REQUEST_METHOD, "POST");
+
+                final FullHttpResponse response = client.send(remoteAddress.address(), request);
+                try {
+                    assertThat(response.status(), equalTo(HttpResponseStatus.OK));
+                    assertThat(response.headers().get(CorsHandler.ACCESS_CONTROL_ALLOW_ORIGIN), equalTo("test-cors.org"));
+                    assertThat(response.headers().get(CorsHandler.VARY), equalTo(CorsHandler.ORIGIN));
+                    assertTrue(response.headers().contains(CorsHandler.DATE));
+                } finally {
+                    response.release();
+                }
+            }
+
+            // Test short-circuited request
+            try (ReactorHttpClient client = ReactorHttpClient.https()) {
+                final FullHttpRequest request = new DefaultFullHttpRequest(HttpVersion.HTTP_1_1, HttpMethod.GET, "/");
+                request.headers().add(CorsHandler.ORIGIN, "google.com");
+
+                final FullHttpResponse response = client.send(remoteAddress.address(), request);
+                try {
+                    assertThat(response.status(), equalTo(HttpResponseStatus.FORBIDDEN));
+                } finally {
+                    response.release();
+                }
+            }
+        }
+    }
+
+    public void testConnectTimeout() throws Exception {
+        final HttpServerTransport.Dispatcher dispatcher = new HttpServerTransport.Dispatcher() {
+
+            @Override
+            public void dispatchRequest(final RestRequest request, final RestChannel channel, final ThreadContext threadContext) {
+                logger.error("--> Unexpected successful request [{}]", FakeRestRequest.requestToString(request));
+                throw new AssertionError("Should not have received a dispatched request");
+            }
+
+            @Override
+            public void dispatchBadRequest(final RestChannel channel, final ThreadContext threadContext, final Throwable cause) {
+                logger.error(
+                    new ParameterizedMessage("--> Unexpected bad request [{}]", FakeRestRequest.requestToString(channel.request())),
+                    cause
+                );
+                throw new AssertionError("Should not have received a dispatched request");
+            }
+
+        };
+
+        Settings settings = createBuilderWithPort().put(
+            HttpTransportSettings.SETTING_HTTP_CONNECT_TIMEOUT.getKey(),
+            new TimeValue(randomIntBetween(100, 300))
+        ).build();
+
+        NioEventLoopGroup group = new NioEventLoopGroup();
+        try (
+            ReactorNetty4HttpServerTransport transport = new ReactorNetty4HttpServerTransport(
+                settings,
+                networkService,
+                bigArrays,
+                threadPool,
+                xContentRegistry(),
+                dispatcher,
+                new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
+                new SharedGroupFactory(settings),
+                secureHttpTransportSettingsProvider,
+                NoopTracer.INSTANCE
+            )
+        ) {
+            transport.start();
+            final TransportAddress remoteAddress = randomFrom(transport.boundAddress().boundAddresses());
+
+            final CountDownLatch channelClosedLatch = new CountDownLatch(1);
+
+            final Bootstrap clientBootstrap = new Bootstrap().option(ChannelOption.ALLOCATOR, NettyAllocator.getAllocator())
+                .channel(NioSocketChannel.class)
+                .handler(new ChannelInitializer<SocketChannel>() {
+
+                    @Override
+                    protected void initChannel(SocketChannel ch) {
+                        ch.pipeline().addLast(new ChannelHandlerAdapter() {
+                        });
+
+                    }
+                })
+                .group(group);
+            ChannelFuture connect = clientBootstrap.connect(remoteAddress.address());
+            connect.channel().closeFuture().addListener(future -> channelClosedLatch.countDown());
+
+            assertTrue("Channel should be closed due to read timeout", channelClosedLatch.await(1, TimeUnit.MINUTES));
+
+        } finally {
+            group.shutdownGracefully().await();
+        }
+    }
+
+    private Settings createSettings() {
+        return createBuilderWithPort().build();
+    }
+
+    private Settings.Builder createBuilderWithPort() {
+        return Settings.builder().put(HttpTransportSettings.SETTING_HTTP_PORT.getKey(), getPortRange());
+    }
+}
diff --git a/plugins/transport-reactor-netty4/src/test/resources/README.txt b/plugins/transport-reactor-netty4/src/test/resources/README.txt
new file mode 100644
index 0000000000000..a4353cee45a97
--- /dev/null
+++ b/plugins/transport-reactor-netty4/src/test/resources/README.txt
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+#
+# This is README describes how the certificates in this directory were created.
+# This file can also be executed as a script
+#
+
+# 1. Create certificate key
+
+openssl req  -x509  -sha256  -newkey rsa:2048  -keyout certificate.key  -out certificate.crt  -days 1024  -nodes
+
+# 2. Export the certificate in pkcs12 format
+
+openssl pkcs12  -export  -in certificate.crt  -inkey certificate.key  -out server.p12  -name netty4-secure -password pass:password
+
diff --git a/plugins/transport-reactor-netty4/src/test/resources/certificate.crt b/plugins/transport-reactor-netty4/src/test/resources/certificate.crt
new file mode 100644
index 0000000000000..54c78fdbcf6de
--- /dev/null
+++ b/plugins/transport-reactor-netty4/src/test/resources/certificate.crt
@@ -0,0 +1,22 @@
+-----BEGIN CERTIFICATE-----
+MIIDkzCCAnugAwIBAgIUddAawr5zygcd+Dcn9WVDpO4BJ7YwDQYJKoZIhvcNAQEL
+BQAwWTELMAkGA1UEBhMCQVUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM
+GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDESMBAGA1UEAwwJbG9jYWxob3N0MB4X
+DTI0MDMxNDE5NDQzOVoXDTI3MDEwMjE5NDQzOVowWTELMAkGA1UEBhMCQVUxEzAR
+BgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5
+IEx0ZDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A
+MIIBCgKCAQEAzjOKkg6Iba5zfZ8b/RYw+PGmGEfbdGuuF10Wz4Jmx/Nk4VfDLxdh
+TW8VllUL2JD7uPkjABj7pW3awAbvIJ+VGbKqfBr1Nsz0mPPzhT8cfuMH/FDZgQs3
+4HuqDKr0LfC1Kw5E3WF0GVMBDNu0U+nKoeqySeYjGdxDnd3W4cqK5AnUxL0RnIny
+Bw7ZuhcU55XndH/Xauro/2EpvJduDsWMdqt7ZfIf1TOmaiQHK+82yb/drVaJbczK
+uTpn1Kv2bnzkQEckgq+z1dLNOOyvP2xf+nsziw5ilJe92e5GJOUJYFAlEgUAGpfD
+dv6j/gTRYvdJCJItOQEQtektNCAZsoc0wwIDAQABo1MwUTAdBgNVHQ4EFgQUzHts
+wIt+zhB/R4U4Do2P6rr0YhkwHwYDVR0jBBgwFoAUzHtswIt+zhB/R4U4Do2P6rr0
+YhkwDwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAveh870jJX7vt
+oLCrdugsyo79pR4f7Nr1kUy3jJrfoaoUmrjiiiHWgT22fGwp7j1GZF2mVfo8YVaK
+63YNn5gB2NNZhguPOFC4AdvHRYOKRBOaOvWK8oq7BcJ//18JYI/pPnpgkYvJjqv4
+gFKaZX9qWtujHpAmKiVGs7pwYGNXfixPHRNV4owcfHMIH5dhbbqT49j94xVpjbXs
+OymKtFl4kpCE/0LzKFrFcuu55Am1VLBHx2cPpHLOipgUcF5BHFlQ8AXiCMOwfPAw
+d22mLB6Gt1oVEpyvQHYd3e04FetEXQ9E8T+NKWZx/8Ucf+IWBYmZBRxch6O83xgk
+bAbGzqkbzQ==
+-----END CERTIFICATE-----
diff --git a/plugins/transport-reactor-netty4/src/test/resources/certificate.key b/plugins/transport-reactor-netty4/src/test/resources/certificate.key
new file mode 100644
index 0000000000000..228350180935d
--- /dev/null
+++ b/plugins/transport-reactor-netty4/src/test/resources/certificate.key
@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDOM4qSDohtrnN9
+nxv9FjD48aYYR9t0a64XXRbPgmbH82ThV8MvF2FNbxWWVQvYkPu4+SMAGPulbdrA
+Bu8gn5UZsqp8GvU2zPSY8/OFPxx+4wf8UNmBCzfge6oMqvQt8LUrDkTdYXQZUwEM
+27RT6cqh6rJJ5iMZ3EOd3dbhyorkCdTEvRGcifIHDtm6FxTnled0f9dq6uj/YSm8
+l24OxYx2q3tl8h/VM6ZqJAcr7zbJv92tVoltzMq5OmfUq/ZufORARySCr7PV0s04
+7K8/bF/6ezOLDmKUl73Z7kYk5QlgUCUSBQAal8N2/qP+BNFi90kIki05ARC16S00
+IBmyhzTDAgMBAAECggEAVOdiElvLjyX6xeoC00YU6hxOIMdNtHU2HMamwtDV01UD
+38mMQ9KjrQelYt4n34drLrHe2IZw75/5J4JzagJrmUY47psHBwaDXItuZRokeJaw
+zhLYTEs7OcKRtV+a5WOspUrdzi33aQoFb67zZG3qkpsZyFXrdBV+/fy/Iv+MCvLH
+xR0jQ5mzE3cw20R7S4nddChBA/y8oKGOo6QRf2SznC1jL/+yolHvJPEn1v8AUxYm
+BMPHxj1O0c4M4IxnJQ3Y5Jy9OaFMyMsFlF1hVhc/3LDDxDyOuBsVsFDicojyrRea
+GKngIke0yezy7Wo4NUcp8YQhafonpWVsSJJdOUotcQKBgQD0rihFBXVtcG1d/Vy7
+FvLHrmccD56JNV744LSn2CDM7W1IulNbDUZINdCFqL91u5LpxozeE1FPY1nhwncJ
+N7V7XYCaSLCuV1YJzRmUCjnzk2RyopGpzWog3f9uUFGgrk1HGbNAv99k/REya6Iu
+IRSkuQhaJOj3bRXzonh0K4GjewKBgQDXvamtCioOUMSP8vq919YMkBw7F+z/fr0p
+pamO8HL9eewAUg6N92JQ9kobSo/GptdmdHIjs8LqnS5C3H13GX5Qlf5GskOlCpla
+V55ElaSp0gvKwWE168U7gQH4etPQAXXJrOGFaGbPj9W81hTUud7HVE88KYdfWTBo
+I7TuE25tWQKBgBRjcr2Vn9xXsvVTCGgamG5lLPhcoNREGz7X0pXt34XT/vhBdnKu
+331i5pZMom+YCrzqK5DRwUPBPpseTjb5amj2OKIijn5ojqXQbmI0m/GdBZC71TF2
+CXLlrMQvcy3VeGEFVjd+BYpvwAAYkfIQFZ1IQdbpHnSHpX2guzLK8UmDAoGBANUy
+PIcf0EetUVHfkCIjNQfdMcjD8BTcLhsF9vWmcDxFTA9VB8ULf0D64mjt2f85yQsa
+b+EQN8KZ6alxMxuLOeRxFYLPj0F9o+Y/R8wHBV48kCKhz2r1v0b6SfQ/jSm1B61x
+BrxLW64qOdIOzS8bLyhUDKkrcPesr8V548aRtUKhAoGBAKlNJFd8BCGKD9Td+3dE
+oP1iHTX5XZ+cQIqL0e+GMQlK4HnQP566DFZU5/GHNNAfmyxd5iSRwhTqPMHRAmOb
+pqQwsyufx0dFeIBxeSO3Z6jW5h2sl4nBipZpw9bzv6EBL1xRr0SfMNZzdnf4JFzc
+0htGo/VO93Z2pv8w7uGUz1nN
+-----END PRIVATE KEY-----
diff --git a/qa/wildfly/build.gradle b/qa/wildfly/build.gradle
index 5d37be47e782e..abf033fff378a 100644
--- a/qa/wildfly/build.gradle
+++ b/qa/wildfly/build.gradle
@@ -40,7 +40,7 @@ apply plugin: 'opensearch.internal-distribution-download'
 testFixtures.useFixture()
 
 dependencies {
-  providedCompile('jakarta.enterprise:jakarta.enterprise.cdi-api:4.0.1') {
+  providedCompile('jakarta.enterprise:jakarta.enterprise.cdi-api:4.1.0') {
     exclude module: 'jakarta.annotation-api'
   }
   providedCompile 'jakarta.ws.rs:jakarta.ws.rs-api:3.1.0'
diff --git a/release-notes/opensearch.release-notes-1.3.16.md b/release-notes/opensearch.release-notes-1.3.16.md
new file mode 100644
index 0000000000000..a434e419d5780
--- /dev/null
+++ b/release-notes/opensearch.release-notes-1.3.16.md
@@ -0,0 +1,4 @@
+## 2024-04-18 Version 1.3.16 Release Notes
+
+### Upgrades
+- Bump `netty` from 4.1.107.Final to 4.1.109.Final ([#12924](https://github.com/opensearch-project/OpenSearch/pull/12924), [#13233](https://github.com/opensearch-project/OpenSearch/pull/13233))
diff --git a/release-notes/opensearch.release-notes-2.14.0.md b/release-notes/opensearch.release-notes-2.14.0.md
new file mode 100644
index 0000000000000..8ef0215baa67a
--- /dev/null
+++ b/release-notes/opensearch.release-notes-2.14.0.md
@@ -0,0 +1,87 @@
+## 2024-05-07 Version 2.14.0 Release Notes
+
+## [2.14.0]
+### Added
+- Add explicit dependency to validatePom and generatePom tasks ([#12909](https://github.com/opensearch-project/OpenSearch/pull/12909))
+- Constant Keyword Field ([#12285](https://github.com/opensearch-project/OpenSearch/pull/12285))
+- [Concurrent Segment Search] Perform buildAggregation concurrently and support Composite Aggregations ([#12697](https://github.com/opensearch-project/OpenSearch/pull/12697))
+- Convert ingest processor supports ip type ([#12818](https://github.com/opensearch-project/OpenSearch/pull/12818))
+- Allow setting KEYSTORE_PASSWORD through env variable ([#12865](https://github.com/opensearch-project/OpenSearch/pull/12865))
+- Add a counter to node stat (and _cat/shards) api to track shard going from idle to non-idle ([#12768](https://github.com/opensearch-project/OpenSearch/pull/12768))
+- [Concurrent Segment Search] Disable concurrent segment search for system indices and throttled requests ([#12954](https://github.com/opensearch-project/OpenSearch/pull/12954))
+- Rename ingest processor supports overriding target field if exists ([#12990](https://github.com/opensearch-project/OpenSearch/pull/12990))
+- [Tiered Caching] Make took time caching policy setting dynamic ([#13063](https://github.com/opensearch-project/OpenSearch/pull/13063))
+- Detect breaking changes on pull requests ([#9044](https://github.com/opensearch-project/OpenSearch/pull/9044))
+- Add cluster primary balance contraint for rebalancing with buffer ([#12656](https://github.com/opensearch-project/OpenSearch/pull/12656))
+- [Remote Store] Make translog transfer timeout configurable ([#12704](https://github.com/opensearch-project/OpenSearch/pull/12704))
+- Derived fields support to derive field values at query time without indexing ([#12569](https://github.com/opensearch-project/OpenSearch/pull/12569))
+- Add support for more than one protocol for transport ([#12967](https://github.com/opensearch-project/OpenSearch/pull/12967))
+- [Tiered Caching] Add dimension-based stats to ICache implementations. ([#12531](https://github.com/opensearch-project/OpenSearch/pull/12531))
+- Add changes for overriding remote store and replication settings during snapshot restore. ([#11868](https://github.com/opensearch-project/OpenSearch/pull/11868))
+- Reject Resize index requests (i.e, split, shrink and clone), While DocRep to SegRep migration is in progress.([#12686](https://github.com/opensearch-project/OpenSearch/pull/12686))
+- Add an individual setting of rate limiter for segment replication ([#12959](https://github.com/opensearch-project/OpenSearch/pull/12959))
+- [Tiered Caching] Add dimension-based stats to TieredSpilloverCache ([#13236](https://github.com/opensearch-project/OpenSearch/pull/13236))
+- [Tiered Caching] Expose new cache stats API ([#13237](https://github.com/opensearch-project/OpenSearch/pull/13237))
+- [Streaming Indexing] Ensure support of the new transport by security plugin ([#13174](https://github.com/opensearch-project/OpenSearch/pull/13174))
+- Add cluster setting to dynamically configure the buckets for filter rewrite optimization. ([#13179](https://github.com/opensearch-project/OpenSearch/pull/13179))
+- [Tiered caching] Make Indices Request Cache Stale Key Mgmt Threshold setting dynamic ([#12941](https://github.com/opensearch-project/OpenSearch/pull/12941))
+- Make search query counters dynamic to support all query types ([#12601](https://github.com/opensearch-project/OpenSearch/pull/12601))
+- [Tiered Caching] Gate new stats logic behind FeatureFlags.PLUGGABLE_CACHE ([#13238](https://github.com/opensearch-project/OpenSearch/pull/13238))
+- [Tiered Caching] Add a dynamic setting to disable/enable disk cache. ([#13373](https://github.com/opensearch-project/OpenSearch/pull/13373))
+- Batch mode for async fetching shard information in GatewayAllocator for unassigned shards ([#8746](https://github.com/opensearch-project/OpenSearch/pull/8746))
+- [Remote Store] Add settings for remote path type and hash algorithm ([#13225](https://github.com/opensearch-project/OpenSearch/pull/13225))
+- [Remote Store] Upload remote paths during remote enabled index creation ([#13386](https://github.com/opensearch-project/OpenSearch/pull/13386))
+- [Search Pipeline] Handle default pipeline for multiple indices ([#13276](https://github.com/opensearch-project/OpenSearch/pull/13276))
+- [Batch Ingestion] Add `batch_size` to `_bulk` API. ([#12457](https://github.com/opensearch-project/OpenSearch/issues/12457))
+- [Remote Store] Add capability of doing refresh as determined by the translog ([#12992](https://github.com/opensearch-project/OpenSearch/pull/12992))
+- Support multi ranges traversal when doing date histogram rewrite optimization. ([#13317](https://github.com/opensearch-project/OpenSearch/pull/13317))
+
+### Dependencies
+- Bump `org.apache.commons:commons-configuration2` from 2.10.0 to 2.10.1 ([#12896](https://github.com/opensearch-project/OpenSearch/pull/12896))
+- Bump `asm` from 9.6 to 9.7 ([#12908](https://github.com/opensearch-project/OpenSearch/pull/12908))
+- Bump `net.minidev:json-smart` from 2.5.0 to 2.5.1 ([#12893](https://github.com/opensearch-project/OpenSearch/pull/12893), [#13117](https://github.com/opensearch-project/OpenSearch/pull/13117))
+- Bump `netty` from 4.1.107.Final to 4.1.109.Final ([#12924](https://github.com/opensearch-project/OpenSearch/pull/12924), [#13233](https://github.com/opensearch-project/OpenSearch/pull/13233))
+- Bump `commons-io:commons-io` from 2.15.1 to 2.16.0 ([#12996](https://github.com/opensearch-project/OpenSearch/pull/12996), [#12998](https://github.com/opensearch-project/OpenSearch/pull/12998), [#12999](https://github.com/opensearch-project/OpenSearch/pull/12999))
+- Bump `org.apache.commons:commons-compress` from 1.24.0 to 1.26.1 ([#12627](https://github.com/opensearch-project/OpenSearch/pull/12627))
+- Bump `org.apache.commons:commonscodec` from 1.15 to 1.16.1 ([#12627](https://github.com/opensearch-project/OpenSearch/pull/12627))
+- Bump `org.apache.commons:commonslang` from 3.13.0 to 3.14.0 ([#12627](https://github.com/opensearch-project/OpenSearch/pull/12627))
+- Bump Apache Tika from 2.6.0 to 2.9.2 ([#12627](https://github.com/opensearch-project/OpenSearch/pull/12627))
+- Bump `com.gradle.enterprise` from 3.16.2 to 3.17.2 ([#13116](https://github.com/opensearch-project/OpenSearch/pull/13116), [#13191](https://github.com/opensearch-project/OpenSearch/pull/13191), [#13327](https://github.com/opensearch-project/OpenSearch/pull/13327))
+- Bump `gradle/wrapper-validation-action` from 2 to 3 ([#13192](https://github.com/opensearch-project/OpenSearch/pull/13192))
+- Bump joda from 2.12.2 to 2.12.7 ([#13193](https://github.com/opensearch-project/OpenSearch/pull/13193))
+- Bump bouncycastle from 1.77 to 1.78 ([#13243](https://github.com/opensearch-project/OpenSearch/pull/13243))
+- Update google dependencies in repository-gcs and discovery-gce ([#13213](https://github.com/opensearch-project/OpenSearch/pull/13213))
+- Bump `com.google.apis:google-api-services-compute` from v1-rev235-1.25.0 to v1-rev20240407-2.0.0 ([#13333](https://github.com/opensearch-project/OpenSearch/pull/13333))
+- Bump `com.github.spullara.mustache.java:compiler` from 0.9.10 to 0.9.11 ([#13329](https://github.com/opensearch-project/OpenSearch/pull/13329))
+- Bump `commons-cli:commons-cli` from 1.6.0 to 1.7.0 ([#13331](https://github.com/opensearch-project/OpenSearch/pull/13331))
+- Bump `jakarta.enterprise:jakarta.enterprise.cdi-api` from 4.0.1 to 4.1.0 ([#13328](https://github.com/opensearch-project/OpenSearch/pull/13328))
+- Bump `com.google.api.grpc:proto-google-iam-v1` from 0.12.0 to 1.33.0 ([#13332](https://github.com/opensearch-project/OpenSearch/pull/13332))
+- Bump `com.squareup.okio:okio` from 3.8.0 to 3.9.0 ([#12997](https://github.com/opensearch-project/OpenSearch/pull/12997))
+- OpenJDK Update (April 2024 Patch releases), update to Eclipse Temurin 21.0.3+9 ([#13389](https://github.com/opensearch-project/OpenSearch/pull/13389))
+- Bump `org.bouncycastle:bc-fips` from 1.0.2.4 to 1.0.2.5 ([#13446](https://github.com/opensearch-project/OpenSearch/pull/13446))
+- Bump `lycheeverse/lychee-action` from 1.9.3 to 1.10.0 ([#13447](https://github.com/opensearch-project/OpenSearch/pull/13447))
+- Bump `com.netflix.nebula.ospackage-base` from 11.8.1 to 11.9.0 ([#13440](https://github.com/opensearch-project/OpenSearch/pull/13440))
+
+### Changed
+- [BWC and API enforcement] Enforcing the presence of API annotations at build time ([#12872](https://github.com/opensearch-project/OpenSearch/pull/12872))
+- Improve built-in secure transports support ([#12907](https://github.com/opensearch-project/OpenSearch/pull/12907))
+- Update links to documentation in rest-api-spec ([#13043](https://github.com/opensearch-project/OpenSearch/pull/13043))
+- Ignoring unavailable shards during search request execution with ignore_available parameter ([#13298](https://github.com/opensearch-project/OpenSearch/pull/13298))
+- Refactoring globMatch using simpleMatchWithNormalizedStrings from Regex ([#13104](https://github.com/opensearch-project/OpenSearch/pull/13104))
+- [BWC and API enforcement] Reconsider the breaking changes check policy to detect breaking changes against released versions ([#13292](https://github.com/opensearch-project/OpenSearch/pull/13292))
+- Switch to macos-13 runner for precommit and assemble github actions due to macos-latest is now arm64 ([#13412](https://github.com/opensearch-project/OpenSearch/pull/13412))
+- [Revert] Prevent unnecessary fetch sub phase processor initialization during fetch phase execution ([#12503](https://github.com/opensearch-project/OpenSearch/pull/12503))
+
+### Fixed
+- Fix bulk API ignores ingest pipeline for upsert ([#12883](https://github.com/opensearch-project/OpenSearch/pull/12883))
+- Fix issue with feature flags where default value may not be honored ([#12849](https://github.com/opensearch-project/OpenSearch/pull/12849))
+- Fix UOE While building Exists query for nested search_as_you_type field ([#12048](https://github.com/opensearch-project/OpenSearch/pull/12048))
+- Enabled mockTelemetryPlugin for IT and fixed OOM issues ([#13054](https://github.com/opensearch-project/OpenSearch/pull/13054))
+- Fix from and size parameter can be negative when searching ([#13047](https://github.com/opensearch-project/OpenSearch/pull/13047))
+- Fix implement mark() and markSupported() in class FilterStreamInput ([#13098](https://github.com/opensearch-project/OpenSearch/pull/13098))
+- Fix IndicesRequestCache Stale calculation ([#13070](https://github.com/opensearch-project/OpenSearch/pull/13070)]
+- Fix snapshot _status API to return correct status for partial snapshots ([#12812](https://github.com/opensearch-project/OpenSearch/pull/12812))
+- Improve the error messages for _stats with closed indices ([#13012](https://github.com/opensearch-project/OpenSearch/pull/13012))
+- Ignore BaseRestHandler unconsumed content check as it's always consumed. ([#13290](https://github.com/opensearch-project/OpenSearch/pull/13290))
+- Fix mapper_parsing_exception when using flat_object fields with names longer than 11 characters ([#13259](https://github.com/opensearch-project/OpenSearch/pull/13259))
+- DATETIME_FORMATTER_CACHING_SETTING experimental feature should not default to 'true' ([#13532](https://github.com/opensearch-project/OpenSearch/pull/13532))
\ No newline at end of file
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/bulk.json b/rest-api-spec/src/main/resources/rest-api-spec/api/bulk.json
index bb066cd131480..e0566b811ff07 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/bulk.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/bulk.json
@@ -74,6 +74,10 @@
       "require_alias": {
         "type": "boolean",
         "description": "Sets require_alias for all incoming documents. Defaults to unset (false)"
+      },
+      "batch_size": {
+        "type": "int",
+        "description": "Sets the batch size"
       }
     },
     "body":{
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.aliases.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.aliases.json
index 1ba12b00c8178..b280452e313da 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.aliases.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.aliases.json
@@ -1,7 +1,7 @@
 {
   "cat.aliases":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/cat-alias.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/cat/cat-aliases/",
       "description":"Shows information about currently configured aliases to indices including filter and routing infos."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.allocation.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.allocation.json
index 717c1c49808f6..a56d3e9e8a95e 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.allocation.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.allocation.json
@@ -1,7 +1,7 @@
 {
   "cat.allocation":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/cat-allocation.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/cat/cat-allocation/",
       "description":"Provides a snapshot of how many shards are allocated to each data node and how much disk space they are using."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.cluster_manager.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.cluster_manager.json
index cd96038ad0693..3b999d98eb60d 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.cluster_manager.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.cluster_manager.json
@@ -1,7 +1,7 @@
 {
   "cat.cluster_manager":{
     "documentation":{
-      "url":"https://opensearch.org/docs/latest/opensearch/rest-api/cat/cat-master/",
+      "url":"https://opensearch.org/docs/latest/api-reference/cat/cat-cluster_manager/",
       "description":"Returns information about the cluster-manager node."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.count.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.count.json
index 8cfaddf8db83b..e85f399c29fd3 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.count.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.count.json
@@ -1,7 +1,7 @@
 {
   "cat.count":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/cat-count.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/cat/cat-count/",
       "description":"Provides quick access to the document count of the entire cluster, or individual indices."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.fielddata.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.fielddata.json
index 9fbde4736b5ef..f2a3cfa8250cb 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.fielddata.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.fielddata.json
@@ -1,7 +1,7 @@
 {
   "cat.fielddata":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/cat-fielddata.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/cat/cat-field-data/",
       "description":"Shows how much heap memory is currently being used by fielddata on every data node in the cluster."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.help.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.help.json
index 54ab6d6e5168c..c2f1771977cd5 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.help.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.help.json
@@ -1,7 +1,7 @@
 {
   "cat.help":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/cat.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/cat/index/",
       "description":"Returns help for the Cat APIs."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.indices.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.indices.json
index 2491ab309531d..aa36f8fd809aa 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.indices.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.indices.json
@@ -1,7 +1,7 @@
 {
   "cat.indices":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/cat-indices.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/cat/cat-indices/",
       "description":"Returns information about indices: number of primaries and replicas, document counts, disk size, ..."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.nodeattrs.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.nodeattrs.json
index c8afa4cb17039..92a7e14b234ce 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.nodeattrs.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.nodeattrs.json
@@ -1,7 +1,7 @@
 {
   "cat.nodeattrs":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/cat-nodeattrs.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/cat/cat-nodeattrs/",
       "description":"Returns information about custom node attributes."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.nodes.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.nodes.json
index abb13d015b4d5..63c70afcb9465 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.nodes.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.nodes.json
@@ -1,7 +1,7 @@
 {
   "cat.nodes":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/cat-nodes.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/cat/cat-nodes/",
       "description":"Returns basic statistics about performance of cluster nodes."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.pending_tasks.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.pending_tasks.json
index 9c0edf8c53d90..d89ce8af72270 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.pending_tasks.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.pending_tasks.json
@@ -1,7 +1,7 @@
 {
   "cat.pending_tasks":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/cat-pending-tasks.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/cat/cat-pending-tasks/",
       "description":"Returns a concise representation of the cluster pending tasks."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.plugins.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.plugins.json
index 0b5b39b01ee58..5dd76c0dec4e4 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.plugins.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.plugins.json
@@ -1,7 +1,7 @@
 {
   "cat.plugins":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/cat-plugins.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/cat/cat-plugins/",
       "description":"Returns information about installed plugins across nodes node."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.recovery.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.recovery.json
index 7baf0b8ded609..a3dee81e73abc 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.recovery.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.recovery.json
@@ -1,7 +1,7 @@
 {
   "cat.recovery":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/cat-recovery.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/cat/cat-recovery/",
       "description":"Returns information about index shard recoveries, both on-going completed."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.repositories.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.repositories.json
index 58960709a99bb..a8174bbd48e07 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.repositories.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.repositories.json
@@ -1,7 +1,7 @@
 {
   "cat.repositories":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/cat-repositories.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/cat/cat-repositories/",
       "description":"Returns information about snapshot repositories registered in the cluster."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segment_replication.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segment_replication.json
index a815cd5b1101b..91d913d7c9340 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segment_replication.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segment_replication.json
@@ -1,7 +1,7 @@
 {
   "cat.segment_replication":{
     "documentation":{
-      "url":"https://github.com/opensearch-project/documentation-website/issues/2627",
+      "url":"https://opensearch.org/docs/latest/api-reference/cat/cat-segment-replication/",
       "description":"Returns information about both on-going and latest completed Segment Replication events"
     },
     "stability":"experimental",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segments.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segments.json
index 5107353c7b14f..f9b0de3e2eb50 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segments.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.segments.json
@@ -1,7 +1,7 @@
 {
   "cat.segments":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/cat-segments.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/cat/cat-segments/",
       "description":"Provides low-level information about the segments in the shards of an index."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.shards.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.shards.json
index fab381a098e3f..05a04c4b580ec 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.shards.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.shards.json
@@ -1,7 +1,7 @@
 {
   "cat.shards":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/cat-shards.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/cat/cat-shards/",
       "description":"Provides a detailed view of shard allocation on nodes."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.snapshots.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.snapshots.json
index 1320207abfe75..c6eb24a50e435 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.snapshots.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.snapshots.json
@@ -1,7 +1,7 @@
 {
   "cat.snapshots":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/cat-snapshots.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/cat/cat-snapshots/",
       "description":"Returns all snapshots in a specific repository."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.tasks.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.tasks.json
index 384668f839642..806b993849940 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.tasks.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.tasks.json
@@ -1,7 +1,7 @@
 {
   "cat.tasks":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/tasks.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/cat/cat-tasks/",
       "description":"Returns information about the tasks currently executing on one or more nodes in the cluster."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.templates.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.templates.json
index d45593b7bb2c8..2f4a5f6e40186 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.templates.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.templates.json
@@ -1,7 +1,7 @@
 {
   "cat.templates":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/cat-templates.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/cat/cat-templates/",
       "description":"Returns information about existing templates."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.thread_pool.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.thread_pool.json
index 1165703490d1a..e5bffbb43de5b 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cat.thread_pool.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cat.thread_pool.json
@@ -1,7 +1,7 @@
 {
   "cat.thread_pool":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/cat-thread-pool.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/cat/cat-thread-pool/",
       "description":"Returns cluster-wide thread pool statistics per node.\nBy default the active, queue and rejected statistics are returned for all thread pools."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/clear_scroll.json b/rest-api-spec/src/main/resources/rest-api-spec/api/clear_scroll.json
index b0e50045cd7cd..ed158978e44a4 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/clear_scroll.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/clear_scroll.json
@@ -1,7 +1,7 @@
 {
   "clear_scroll":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/current/clear-scroll-api.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/scroll/",
       "description":"Explicitly clears the search context for a scroll."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.allocation_explain.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.allocation_explain.json
index e46218a781e1b..2f5985faa5f63 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.allocation_explain.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.allocation_explain.json
@@ -1,7 +1,7 @@
 {
   "cluster.allocation_explain":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-allocation-explain.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/cluster-api/cluster-allocation/",
       "description":"Provides explanations for shard allocations in the cluster."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.delete_component_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.delete_component_template.json
index 43e14ad0e2dd8..737716cec2527 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.delete_component_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.delete_component_template.json
@@ -1,7 +1,7 @@
 {
   "cluster.delete_component_template":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-component-template.html",
+      "url":"https://opensearch.org/docs/latest/im-plugin/index-templates/#create-a-component-template",
       "description":"Deletes a component template"
     },
     "stability":"experimental",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.delete_decommission_awareness.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.delete_decommission_awareness.json
index 13ea101169e60..24e9a59baa742 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.delete_decommission_awareness.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.delete_decommission_awareness.json
@@ -1,7 +1,7 @@
 {
   "cluster.delete_decommission_awareness": {
     "documentation": {
-      "url": "https://opensearch.org/docs/latest/opensearch/rest-api/decommission/",
+      "url": "https://opensearch.org/docs/latest/api-reference/cluster-api/cluster-decommission/",
       "description": "Delete any existing decommission."
     },
     "stability": "experimental",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.delete_weighted_routing.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.delete_weighted_routing.json
index 2cd4081b645e8..a6b09e34e488e 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.delete_weighted_routing.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.delete_weighted_routing.json
@@ -1,7 +1,7 @@
 {
   "cluster.delete_weighted_routing": {
     "documentation": {
-      "url": "https://opensearch.org/docs/latest/opensearch/rest-api/weighted-routing/delete",
+      "url": "https://opensearch.org/docs/latest/api-reference/cluster-api/cluster-awareness",
       "description": "Delete weighted shard routing weights"
     },
     "stability": "stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.get_decommission_awareness.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.get_decommission_awareness.json
index 302dea4ec31a7..feb760c2d7783 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.get_decommission_awareness.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.get_decommission_awareness.json
@@ -1,7 +1,7 @@
 {
   "cluster.get_decommission_awareness": {
     "documentation": {
-      "url": "https://opensearch.org/docs/latest/opensearch/rest-api/decommission/",
+      "url": "https://opensearch.org/docs/latest/api-reference/cluster-api/cluster-decommission/",
       "description": "Get details and status of decommissioned attribute"
     },
     "stability": "experimental",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.get_settings.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.get_settings.json
index c60230dbc43b3..df3e6adb6596d 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.get_settings.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.get_settings.json
@@ -1,7 +1,7 @@
 {
   "cluster.get_settings":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-update-settings.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/cluster-api/cluster-settings/",
       "description":"Returns cluster settings."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.get_weighted_routing.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.get_weighted_routing.json
index 45eb3d2b62a84..c800314f1297e 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.get_weighted_routing.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.get_weighted_routing.json
@@ -1,7 +1,7 @@
 {
   "cluster.get_weighted_routing": {
     "documentation": {
-      "url": "https://opensearch.org/docs/latest/opensearch/rest-api/weighted-routing/get",
+      "url": "https://opensearch.org/docs/latest/api-reference/cluster-api/cluster-awareness/",
       "description": "Fetches weighted shard routing weights"
     },
     "stability": "stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.health.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.health.json
index b96340d682546..db820bc4b2c85 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.health.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.health.json
@@ -1,7 +1,7 @@
 {
   "cluster.health":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-health.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/cluster-api/cluster-health/",
       "description":"Returns basic information about the health of the cluster."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.put_component_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.put_component_template.json
index 05558bc7bfc50..5ac77f0f5eeb1 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.put_component_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.put_component_template.json
@@ -1,7 +1,7 @@
 {
   "cluster.put_component_template":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-component-template.html",
+      "url":"https://opensearch.org/docs/latest/im-plugin/index-templates/#create-a-component-template",
       "description":"Creates or updates a component template"
     },
     "stability":"experimental",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.put_decommission_awareness.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.put_decommission_awareness.json
index bf4ffd454d9df..9bb0b4e96ff2b 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.put_decommission_awareness.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.put_decommission_awareness.json
@@ -1,7 +1,7 @@
 {
   "cluster.put_decommission_awareness": {
     "documentation": {
-      "url": "https://opensearch.org/docs/latest/opensearch/rest-api/decommission/",
+      "url": "https://opensearch.org/docs/latest/api-reference/cluster-api/cluster-decommission/",
       "description": "Decommissions an awareness attribute"
     },
     "stability": "experimental",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.put_settings.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.put_settings.json
index 1e36acc51544d..eec2a24057095 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.put_settings.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.put_settings.json
@@ -1,7 +1,7 @@
 {
   "cluster.put_settings":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-update-settings.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/cluster-api/cluster-settings/",
       "description":"Updates the cluster settings."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.put_weighted_routing.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.put_weighted_routing.json
index 88498517ba336..f22402def2fc4 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.put_weighted_routing.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.put_weighted_routing.json
@@ -1,7 +1,7 @@
 {
   "cluster.put_weighted_routing": {
     "documentation": {
-      "url": "https://opensearch.org/docs/latest/opensearch/rest-api/weighted-routing/put",
+      "url": "https://opensearch.org/docs/latest/api-reference/cluster-api/cluster-awareness",
       "description": "Updates weighted shard routing weights"
     },
     "stability": "stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.remote_info.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.remote_info.json
index 4eac0b55ce6f1..32315ec39f1e4 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.remote_info.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.remote_info.json
@@ -1,7 +1,7 @@
 {
   "cluster.remote_info":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-remote-info.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/remote-info/",
       "description":"Returns the information about configured remote clusters."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.state.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.state.json
index b43ab901785bd..b4c7240548a79 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.state.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.state.json
@@ -1,7 +1,7 @@
 {
   "cluster.state":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-state.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/count/",
       "description":"Returns a comprehensive information about the state of the cluster."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.stats.json b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.stats.json
index f36db0979f4f7..aaaab17fb891a 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.stats.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/cluster.stats.json
@@ -1,7 +1,7 @@
 {
   "cluster.stats":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-stats.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/cluster-api/cluster-stats/",
       "description":"Returns high-level overview of cluster statistics."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/count.json b/rest-api-spec/src/main/resources/rest-api-spec/api/count.json
index 8cdb3db7c12cd..38eb435e29a9e 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/count.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/count.json
@@ -1,7 +1,7 @@
 {
   "count":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/search-count.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/count/",
       "description":"Returns number of documents matching a query."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/create.json b/rest-api-spec/src/main/resources/rest-api-spec/api/create.json
index 767af84b82258..b3e678ace4751 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/create.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/create.json
@@ -1,7 +1,7 @@
 {
   "create":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/docs-index_.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/document-apis/index-document/",
       "description":"Creates a new document in the index.\n\nReturns a 409 response when a document with a same ID already exists in the index."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/create_pit.json b/rest-api-spec/src/main/resources/rest-api-spec/api/create_pit.json
index d3a2104c01bc0..81fb3f7c3b973 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/create_pit.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/create_pit.json
@@ -2,7 +2,7 @@
 {
   "create_pit":{
     "documentation":{
-      "url":"https://opensearch.org/docs/latest/opensearch/rest-api/point_in_time/",
+      "url":"https://opensearch.org/docs/latest/search-plugins/searching-data/point-in-time-api",
       "description":"Creates point in time context."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/dangling_indices.delete_dangling_index.json b/rest-api-spec/src/main/resources/rest-api-spec/api/dangling_indices.delete_dangling_index.json
index 5d832fc794f4f..d0c4aaefbcadf 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/dangling_indices.delete_dangling_index.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/dangling_indices.delete_dangling_index.json
@@ -1,7 +1,7 @@
 {
   "dangling_indices.delete_dangling_index": {
     "documentation": {
-      "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/modules-gateway-dangling-indices.html",
+      "url": "https://opensearch.org/docs/latest/api-reference/index-apis/dangling-index/",
       "description": "Deletes the specified dangling index"
     },
     "stability": "stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/dangling_indices.import_dangling_index.json b/rest-api-spec/src/main/resources/rest-api-spec/api/dangling_indices.import_dangling_index.json
index 5b056e1fa145f..f6cedbe41ef53 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/dangling_indices.import_dangling_index.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/dangling_indices.import_dangling_index.json
@@ -1,7 +1,7 @@
 {
   "dangling_indices.import_dangling_index": {
     "documentation": {
-      "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/modules-gateway-dangling-indices.html",
+      "url": "https://opensearch.org/docs/latest/api-reference/index-apis/dangling-index/",
       "description": "Imports the specified dangling index"
     },
     "stability": "stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/dangling_indices.list_dangling_indices.json b/rest-api-spec/src/main/resources/rest-api-spec/api/dangling_indices.list_dangling_indices.json
index dfc21f56ddfac..0d14c8380c7fe 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/dangling_indices.list_dangling_indices.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/dangling_indices.list_dangling_indices.json
@@ -1,7 +1,7 @@
 {
   "dangling_indices.list_dangling_indices": {
     "documentation": {
-      "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/modules-gateway-dangling-indices.html",
+      "url": "https://opensearch.org/docs/latest/api-reference/index-apis/dangling-index/",
       "description": "Returns all dangling indices."
     },
     "stability": "stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/delete.json b/rest-api-spec/src/main/resources/rest-api-spec/api/delete.json
index 76dceb455627f..668976d360999 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/delete.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/delete.json
@@ -1,7 +1,7 @@
 {
   "delete":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/docs-delete.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/document-apis/delete-document/",
       "description":"Removes a document from the index."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/delete_all_pits.json b/rest-api-spec/src/main/resources/rest-api-spec/api/delete_all_pits.json
index 5ff01aa746df9..1e329e3c27b5b 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/delete_all_pits.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/delete_all_pits.json
@@ -1,7 +1,7 @@
 {
   "delete_all_pits":{
     "documentation":{
-      "url":"https://opensearch.org/docs/latest/opensearch/rest-api/point_in_time/",
+      "url":"https://opensearch.org/docs/latest/search-plugins/searching-data/point-in-time-api",
       "description":"Deletes all active point in time searches."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query.json b/rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query.json
index 4c32974583aac..0c493a092e244 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query.json
@@ -1,7 +1,7 @@
 {
   "delete_by_query":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/docs-delete-by-query.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/document-apis/delete-by-query/",
       "description":"Deletes documents matching the provided query."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query_rethrottle.json b/rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query_rethrottle.json
index 112bfc8a7d2e0..c9660fe2e4427 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query_rethrottle.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/delete_by_query_rethrottle.json
@@ -1,7 +1,7 @@
 {
   "delete_by_query_rethrottle":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-delete-by-query.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/document-apis/delete-by-query/",
       "description":"Changes the number of requests per second for a particular Delete By Query operation."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/delete_pit.json b/rest-api-spec/src/main/resources/rest-api-spec/api/delete_pit.json
index b54d9f76204f4..234001349d889 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/delete_pit.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/delete_pit.json
@@ -1,7 +1,7 @@
 {
   "delete_pit":{
     "documentation":{
-      "url":"https://opensearch.org/docs/latest/opensearch/rest-api/point_in_time/",
+      "url":"https://opensearch.org/docs/latest/search-plugins/searching-data/point-in-time-api",
       "description":"Deletes one or more point in time searches based on the IDs passed."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/delete_script.json b/rest-api-spec/src/main/resources/rest-api-spec/api/delete_script.json
index acaa389738606..b858f71eb2812 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/delete_script.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/delete_script.json
@@ -1,7 +1,7 @@
 {
   "delete_script":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/modules-scripting.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/script-apis/delete-script/",
       "description":"Deletes a script."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/exists.json b/rest-api-spec/src/main/resources/rest-api-spec/api/exists.json
index fd221b474a070..0b4f588a673ac 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/exists.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/exists.json
@@ -1,7 +1,7 @@
 {
   "exists":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/docs-get.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/document-apis/get-documents/",
       "description":"Returns information about whether a document exists in an index."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/exists_source.json b/rest-api-spec/src/main/resources/rest-api-spec/api/exists_source.json
index bdbf818fb5d81..5c44f57c289b5 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/exists_source.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/exists_source.json
@@ -1,7 +1,7 @@
 {
   "exists_source":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/docs-get.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/document-apis/get-documents/",
       "description":"Returns information about whether a document source exists in an index."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/explain.json b/rest-api-spec/src/main/resources/rest-api-spec/api/explain.json
index 7f630f7666f30..4ad263262fe14 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/explain.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/explain.json
@@ -1,7 +1,7 @@
 {
   "explain":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/search-explain.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/explain/",
       "description":"Returns information about why a specific matches (or doesn't match) a query."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/get.json b/rest-api-spec/src/main/resources/rest-api-spec/api/get.json
index 2ce77f17aff10..990ec4685fa54 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/get.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/get.json
@@ -1,7 +1,7 @@
 {
   "get":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/docs-get.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/document-apis/get-documents/",
       "description":"Returns a document."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/get_all_pits.json b/rest-api-spec/src/main/resources/rest-api-spec/api/get_all_pits.json
index 544a8cb11b002..9176d499e754b 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/get_all_pits.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/get_all_pits.json
@@ -1,7 +1,7 @@
 {
   "get_all_pits":{
     "documentation":{
-      "url":"https://opensearch.org/docs/latest/opensearch/rest-api/point_in_time/",
+      "url":"https://opensearch.org/docs/latest/search-plugins/searching-data/point-in-time-api",
       "description":"Lists all active point in time searches."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/get_script_context.json b/rest-api-spec/src/main/resources/rest-api-spec/api/get_script_context.json
index aa770ee9d9f2e..c5034e48e6e65 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/get_script_context.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/get_script_context.json
@@ -1,7 +1,7 @@
 {
   "get_script_context":{
     "documentation":{
-      "url": "https://www.elastic.co/guide/en/elasticsearch/painless/master/painless-contexts.html",
+      "url": "https://opensearch.org/docs/latest/api-reference/script-apis/get-script-language/",
       "description":"Returns all script contexts."
     },
     "stability":"experimental",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/get_script_languages.json b/rest-api-spec/src/main/resources/rest-api-spec/api/get_script_languages.json
index a5e06cb88901b..006f34a9762b0 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/get_script_languages.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/get_script_languages.json
@@ -1,7 +1,7 @@
 {
   "get_script_languages":{
     "documentation":{
-      "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/modules-scripting.html",
+      "url": "https://opensearch.org/docs/latest/api-reference/script-apis/get-script-language/",
       "description":"Returns available script types, languages and contexts"
     },
     "stability":"experimental",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/get_source.json b/rest-api-spec/src/main/resources/rest-api-spec/api/get_source.json
index ad79678388590..ab269fcc0980f 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/get_source.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/get_source.json
@@ -1,7 +1,7 @@
 {
   "get_source":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/docs-get.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/document-apis/get-documents/",
       "description":"Returns the source of a document."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/index.json b/rest-api-spec/src/main/resources/rest-api-spec/api/index.json
index b4865403331b0..a02e5bbd2c6e8 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/index.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/index.json
@@ -1,7 +1,7 @@
 {
   "index":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/docs-index_.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/document-apis/index-document/",
       "description":"Creates or updates a document in an index."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.analyze.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.analyze.json
index aa8e84c1985d6..ddf6b93bba0c5 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.analyze.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.analyze.json
@@ -1,7 +1,7 @@
 {
   "indices.analyze":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-analyze.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/analyze-apis/",
       "description":"Performs the analysis process on a text and return the tokens breakdown of the text."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.clear_cache.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.clear_cache.json
index 0c7eca8c8e6f5..09ff5af473335 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.clear_cache.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.clear_cache.json
@@ -1,7 +1,7 @@
 {
   "indices.clear_cache":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-clearcache.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/index-apis/clear-index-cache/",
       "description":"Clears all or specific caches for one or more indices."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.clone.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.clone.json
index 2d874f4933768..0872e1b0d6266 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.clone.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.clone.json
@@ -1,7 +1,7 @@
 {
   "indices.clone": {
     "documentation": {
-      "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-clone-index.html",
+      "url": "https://opensearch.org/docs/latest/api-reference/index-apis/clone/",
       "description": "Clones an index"
     },
     "stability": "stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.close.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.close.json
index 1182b73541f93..863080befd3d6 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.close.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.close.json
@@ -1,7 +1,7 @@
 {
   "indices.close":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-open-close.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/index-apis/close-index/",
       "description":"Closes an index."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.create.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.create.json
index 53ea4cbd80803..c35876c9e6f15 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.create.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.create.json
@@ -1,7 +1,7 @@
 {
   "indices.create":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-create-index.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/index-apis/create-index/",
       "description":"Creates an index with optional settings and mappings."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.data_streams_stats.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.data_streams_stats.json
index 67f90c48eb79f..4dea89d4189b1 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.data_streams_stats.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.data_streams_stats.json
@@ -1,7 +1,7 @@
 {
   "indices.data_streams_stats":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/data-streams.html",
+      "url":"https://opensearch.org/docs/latest/im-plugin/data-streams/",
       "description":"Provides statistics on operations happening in a data stream."
     },
     "stability":"experimental",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete.json
index 53fdf44bb36a1..ea8a3f6b769e9 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete.json
@@ -1,7 +1,7 @@
 {
   "indices.delete":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-delete-index.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/index-apis/delete-index/",
       "description":"Deletes an index."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_data_stream.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_data_stream.json
index f824fb5207d46..e695fb4cfcf60 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_data_stream.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_data_stream.json
@@ -1,7 +1,7 @@
 {
   "indices.delete_data_stream":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/data-streams.html",
+      "url":"https://opensearch.org/docs/latest/im-plugin/data-streams/",
       "description":"Deletes a data stream."
     },
     "stability":"experimental",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_template.json
index 74dbb1822b64a..252ab47a8df6f 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.delete_template.json
@@ -1,7 +1,7 @@
 {
   "indices.delete_template":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-templates.html",
+      "url":"https://opensearch.org/docs/latest/im-plugin/index-templates",
       "description":"Deletes an index template."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists.json
index 7539f44a81eed..7be004cd9c4ea 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists.json
@@ -1,7 +1,7 @@
 {
   "indices.exists":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-exists.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/index-apis/exists/",
       "description":"Returns information about whether a particular index exists."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_alias.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_alias.json
index 66e5ce92cbbe5..88d1c00076757 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_alias.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_alias.json
@@ -1,7 +1,7 @@
 {
   "indices.exists_alias":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-aliases.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/index-apis/alias/",
       "description":"Returns information about whether a particular alias exists."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_index_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_index_template.json
index c5312680fa880..8b050881240da 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_index_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_index_template.json
@@ -1,7 +1,7 @@
 {
   "indices.exists_index_template":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-templates.html",
+      "url":"https://opensearch.org/docs/latest/im-plugin/index-templates",
       "description":"Returns information about whether a particular index template exists."
     },
     "stability":"experimental",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_template.json
index 9796bdd9d21ff..08ab5cf81e57d 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.exists_template.json
@@ -1,7 +1,7 @@
 {
   "indices.exists_template":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-templates.html",
+      "url":"https://opensearch.org/docs/latest/im-plugin/index-templates",
       "description":"Returns information about whether a particular index template exists."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.forcemerge.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.forcemerge.json
index 986bce55f41e5..a1e066c558cb2 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.forcemerge.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.forcemerge.json
@@ -1,7 +1,7 @@
 {
   "indices.forcemerge":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-forcemerge.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/index-apis/force-merge/",
       "description":"Performs the force merge operation on one or more indices."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get.json
index 0a43f6481d86d..ca701cc81b0c7 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get.json
@@ -1,7 +1,7 @@
 {
   "indices.get":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-get-index.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/index-apis/get-index/",
       "description":"Returns information about one or more indices."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_stream.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_stream.json
index ce19186bea6a9..99f419e593a6f 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_stream.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_data_stream.json
@@ -1,7 +1,7 @@
 {
   "indices.get_data_stream":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/data-streams.html",
+      "url":"https://opensearch.org/docs/latest/im-plugin/data-streams/",
       "description":"Returns data streams."
     },
     "stability":"experimental",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_field_mapping.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_field_mapping.json
index 0e71b6d395777..ef627502f5250 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_field_mapping.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_field_mapping.json
@@ -1,7 +1,7 @@
 {
   "indices.get_field_mapping":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-get-field-mapping.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/index-apis/put-mapping/",
       "description":"Returns mapping for one or more fields."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_index_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_index_template.json
index fbd03f99d2547..3d10894a787f9 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_index_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_index_template.json
@@ -1,7 +1,7 @@
 {
   "indices.get_index_template":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-templates.html",
+      "url":"https://opensearch.org/docs/latest/im-plugin/index-templates",
       "description":"Returns an index template."
     },
     "stability":"experimental",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_mapping.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_mapping.json
index 321bfaba4f941..5f401872c98c4 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_mapping.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_mapping.json
@@ -1,7 +1,7 @@
 {
   "indices.get_mapping":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-get-mapping.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/index-apis/put-mapping/",
       "description":"Returns mappings for one or more indices."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_settings.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_settings.json
index 1bdaea01f87bf..b3db1ca1cbf90 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_settings.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_settings.json
@@ -1,7 +1,7 @@
 {
   "indices.get_settings":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-get-settings.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/index-apis/get-settings/",
       "description":"Returns settings for one or more indices."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_template.json
index 52aeb17913db4..7df19812f9d6b 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.get_template.json
@@ -1,7 +1,7 @@
 {
   "indices.get_template":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-templates.html",
+      "url":"https://opensearch.org/docs/latest/im-plugin/index-templates",
       "description":"Returns an index template."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.open.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.open.json
index f44fb04102a7f..e7fe207a254f6 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.open.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.open.json
@@ -1,7 +1,7 @@
 {
   "indices.open":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-open-close.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/index-apis/open-index/",
       "description":"Opens an index."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_alias.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_alias.json
index 00767afbaec04..c3ccd25da9f86 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_alias.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_alias.json
@@ -1,7 +1,7 @@
 {
   "indices.put_alias":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-aliases.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/index-apis/alias/",
       "description":"Creates or updates an alias."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_index_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_index_template.json
index a2ceb259a4376..439f96943db04 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_index_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_index_template.json
@@ -1,7 +1,7 @@
 {
   "indices.put_index_template":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-templates.html",
+      "url":"https://opensearch.org/docs/latest/im-plugin/index-templates",
       "description":"Creates or updates an index template."
     },
     "stability":"experimental",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_mapping.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_mapping.json
index c8b63d4e1cee1..2db333dc1e8ef 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_mapping.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_mapping.json
@@ -1,7 +1,7 @@
 {
   "indices.put_mapping":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-put-mapping.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/index-apis/put-mapping/",
       "description":"Updates the index mappings."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_template.json
index 3b1c230178bb8..bdedd5519076d 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.put_template.json
@@ -1,7 +1,7 @@
 {
   "indices.put_template":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-templates.html",
+      "url":"https://opensearch.org/docs/latest/im-plugin/index-templates",
       "description":"Creates or updates an index template."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.shrink.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.shrink.json
index a20014a1444ec..941ea1127954d 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.shrink.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.shrink.json
@@ -1,7 +1,7 @@
 {
   "indices.shrink":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-shrink-index.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/index-apis/shrink-index/",
       "description":"Allow to shrink an existing index into a new index with fewer primary shards."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.simulate_index_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.simulate_index_template.json
index 0e42ba6028a9f..1ddb4e54d8bbd 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.simulate_index_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.simulate_index_template.json
@@ -1,7 +1,7 @@
 {
   "indices.simulate_index_template":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-templates.html",
+      "url":"https://opensearch.org/docs/latest/im-plugin/index-templates",
       "description": "Simulate matching the given index name against the index templates in the system"
     },
     "stability":"experimental",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.simulate_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.simulate_template.json
index 65b555082c3b1..04a33fe62e1ec 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.simulate_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.simulate_template.json
@@ -1,7 +1,7 @@
 {
   "indices.simulate_template":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-templates.html",
+      "url":"https://opensearch.org/docs/latest/im-plugin/index-templates",
       "description": "Simulate resolving the given template name or body"
     },
     "stability":"experimental",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.split.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.split.json
index d399bf9dbdb8a..af041f426b644 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.split.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.split.json
@@ -1,7 +1,7 @@
 {
   "indices.split":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-split-index.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/index-apis/split/",
       "description":"Allows you to split an existing index into a new index with more primary shards."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.stats.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.stats.json
index 382bb9efde0ff..71ce6dbd443f0 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.stats.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.stats.json
@@ -1,7 +1,7 @@
 {
   "indices.stats":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-stats.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/index-apis/stats/",
       "description":"Provides statistics on operations happening in an index."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.update_aliases.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.update_aliases.json
index c31cb8fe59c0f..467cc444b6e53 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.update_aliases.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.update_aliases.json
@@ -1,7 +1,7 @@
 {
   "indices.update_aliases":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/indices-aliases.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/index-apis/alias/",
       "description":"Updates index aliases."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.delete_pipeline.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.delete_pipeline.json
index 3e40136f556fa..2e615ece82c64 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.delete_pipeline.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.delete_pipeline.json
@@ -1,7 +1,7 @@
 {
   "ingest.delete_pipeline":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/delete-pipeline-api.html",
+      "url":"https://opensearch.org/docs/latest/ingest-pipelines/delete-ingest/",
       "description":"Deletes a pipeline."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.get_pipeline.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.get_pipeline.json
index cde980e67c8c9..498aac4ca5a4a 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.get_pipeline.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.get_pipeline.json
@@ -1,7 +1,7 @@
 {
   "ingest.get_pipeline":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/get-pipeline-api.html",
+      "url":"https://opensearch.org/docs/latest/ingest-pipelines/get-ingest/",
       "description":"Returns a pipeline."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.put_pipeline.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.put_pipeline.json
index 5475905e7b99f..0043a32a55310 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.put_pipeline.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.put_pipeline.json
@@ -1,7 +1,7 @@
 {
   "ingest.put_pipeline":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/put-pipeline-api.html",
+      "url":"https://opensearch.org/docs/latest/ingest-pipelines/create-ingest/",
       "description":"Creates or updates a pipeline."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.simulate.json b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.simulate.json
index 8122f7a0ffa19..6f6b153195255 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.simulate.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/ingest.simulate.json
@@ -1,7 +1,7 @@
 {
   "ingest.simulate":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/simulate-pipeline-api.html",
+      "url":"https://opensearch.org/docs/latest/ingest-pipelines/simulate-ingest/",
       "description":"Allows to simulate a pipeline with example documents."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/msearch.json b/rest-api-spec/src/main/resources/rest-api-spec/api/msearch.json
index 3a3a6ebe1bff5..e4b1c2a798a41 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/msearch.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/msearch.json
@@ -1,7 +1,7 @@
 {
   "msearch":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/search-multi-search.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/multi-search/",
       "description":"Allows to execute several search operations in one request."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/msearch_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/msearch_template.json
index 7ac194f91bf56..42aa8a23b1510 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/msearch_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/msearch_template.json
@@ -1,7 +1,7 @@
 {
   "msearch_template":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/current/search-multi-search.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/search-template/",
       "description":"Allows to execute several search template operations in one request."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.hot_threads.json b/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.hot_threads.json
index 0830344dc4ad4..bbbe79b2693c2 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.hot_threads.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.hot_threads.json
@@ -1,7 +1,7 @@
 {
   "nodes.hot_threads":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-nodes-hot-threads.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/nodes-apis/nodes-hot-threads/",
       "description":"Returns information about hot threads on each node in the cluster."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.info.json b/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.info.json
index 37279edd3106f..4dce8ec54635a 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.info.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.info.json
@@ -1,7 +1,7 @@
 {
   "nodes.info":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-nodes-info.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/nodes-apis/nodes-info/",
       "description":"Returns information about nodes in the cluster."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.reload_secure_settings.json b/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.reload_secure_settings.json
index 25dc72b6cc037..6ecbf96c1925b 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.reload_secure_settings.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.reload_secure_settings.json
@@ -1,7 +1,7 @@
 {
   "nodes.reload_secure_settings":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/secure-settings.html#reloadable-secure-settings",
+      "url":"https://opensearch.org/docs/latest/api-reference/nodes-apis/nodes-reload-secure/",
       "description":"Reloads secure settings."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.stats.json b/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.stats.json
index cc1a9e8185093..a23b2e5428fb6 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.stats.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.stats.json
@@ -1,7 +1,7 @@
 {
   "nodes.stats":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-nodes-stats.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/nodes-apis/nodes-stats/",
       "description":"Returns statistical information about nodes in the cluster."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.usage.json b/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.usage.json
index 5acbf7a51116c..e23011ce432a8 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.usage.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/nodes.usage.json
@@ -1,7 +1,7 @@
 {
   "nodes.usage":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/cluster-nodes-usage.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/nodes-apis/nodes-usage/",
       "description":"Returns low-level information about REST actions usage on nodes."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/put_script.json b/rest-api-spec/src/main/resources/rest-api-spec/api/put_script.json
index c8413d1476402..1c13ea41e3470 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/put_script.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/put_script.json
@@ -1,7 +1,7 @@
 {
   "put_script":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/modules-scripting.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/nodes-apis/nodes-usage/",
       "description":"Creates or updates a script."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/rank_eval.json b/rest-api-spec/src/main/resources/rest-api-spec/api/rank_eval.json
index eadf240192394..04d337a45ec3e 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/rank_eval.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/rank_eval.json
@@ -1,7 +1,7 @@
 {
   "rank_eval":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/search-rank-eval.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/rank-eval/",
       "description":"Allows to evaluate the quality of ranked search results over a set of typical search queries"
     },
     "stability":"experimental",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/reindex.json b/rest-api-spec/src/main/resources/rest-api-spec/api/reindex.json
index 2fbaf86cab616..e2cb76d9ab8ca 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/reindex.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/reindex.json
@@ -1,7 +1,7 @@
 {
   "reindex":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/docs-reindex.html",
+      "url":"https://opensearch.org/docs/latest/im-plugin/reindex-data/",
       "description":"Allows to copy documents from one index to another, optionally filtering the source\ndocuments by a query, changing the destination index settings, or fetching the\ndocuments from a remote cluster."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/reindex_rethrottle.json b/rest-api-spec/src/main/resources/rest-api-spec/api/reindex_rethrottle.json
index d91365b3c49a5..eafd1122f21c2 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/reindex_rethrottle.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/reindex_rethrottle.json
@@ -1,7 +1,7 @@
 {
   "reindex_rethrottle":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/docs-reindex.html",
+      "url":"https://opensearch.org/docs/latest/im-plugin/reindex-data/",
       "description":"Changes the number of requests per second for a particular Reindex operation."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/remote_store.restore.json b/rest-api-spec/src/main/resources/rest-api-spec/api/remote_store.restore.json
index 6af49f75b9f6e..4d8efb9411bff 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/remote_store.restore.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/remote_store.restore.json
@@ -1,7 +1,7 @@
 {
   "remote_store.restore":{
     "documentation":{
-      "url": "https://opensearch.org/docs/latest/opensearch/rest-api/remote-store#restore",
+      "url": "https://opensearch.org/docs/latest/tuning-your-cluster/availability-and-recovery/remote-store/index/#restoring-from-a-backup",
       "description":"Restores from remote store."
     },
     "stability":"experimental",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/remote_store.stats.json b/rest-api-spec/src/main/resources/rest-api-spec/api/remote_store.stats.json
index 437a4439bbcb5..5b456ca35d77a 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/remote_store.stats.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/remote_store.stats.json
@@ -1,7 +1,7 @@
 {
   "remote_store.stats":{
     "documentation":{
-      "url": "https://opensearch.org/docs/latest/tuning-your-cluster/availability-and-recovery/remote",
+      "url": "https://opensearch.org/docs/latest/tuning-your-cluster/availability-and-recovery/remote-store/remote-store-stats-api/",
       "description":"Stats for remote store."
     },
     "stability":"experimental",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/render_search_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/render_search_template.json
index c2c474edd9853..a16839aa83a7d 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/render_search_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/render_search_template.json
@@ -1,7 +1,7 @@
 {
   "render_search_template":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/current/search-template.html#_validating_templates",
+      "url":"https://opensearch.org/docs/latest/api-reference/search-template/",
       "description":"Allows to use the Mustache language to pre-render a search definition."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/scripts_painless_execute.json b/rest-api-spec/src/main/resources/rest-api-spec/api/scripts_painless_execute.json
index 9f761fb452ba1..168620c0d1f3c 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/scripts_painless_execute.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/scripts_painless_execute.json
@@ -1,7 +1,7 @@
 {
   "scripts_painless_execute":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/painless/master/painless-execute-api.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/script-apis/exec-stored-script/",
       "description":"Allows an arbitrary script to be executed and a result to be returned"
     },
     "stability":"experimental",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/scroll.json b/rest-api-spec/src/main/resources/rest-api-spec/api/scroll.json
index ea0cbe2675325..3a4987ff2f6f4 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/scroll.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/scroll.json
@@ -1,7 +1,7 @@
 {
   "scroll":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/search-request-body.html#request-body-search-scroll",
+      "url":"https://opensearch.org/docs/latest/api-reference/scroll/",
       "description":"Allows to retrieve a large numbers of results from a single search request."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json
index e78d49a67a98a..01120eb07250d 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json
@@ -1,7 +1,7 @@
 {
   "search":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/search-search.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/search/",
       "description":"Returns results matching a query."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search_pipeline.delete.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search_pipeline.delete.json
index 1fa7060b974dc..260bf482822cc 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/search_pipeline.delete.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search_pipeline.delete.json
@@ -2,7 +2,7 @@
   "search_pipeline.delete": {
     "documentation": {
       "description": "Deletes a search pipeline.",
-      "url": "https://opensearch.org/docs/latest/opensearch/rest-api/search_pipelines/"
+      "url": "https://opensearch.org/docs/latest/search-plugins/search-pipelines/index/"
     },
     "stability": "stable",
     "url": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search_pipeline.get.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search_pipeline.get.json
index 7cac6e7aa4bcf..05aec5d5de860 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/search_pipeline.get.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search_pipeline.get.json
@@ -2,7 +2,7 @@
   "search_pipeline.get": {
     "documentation": {
       "description": "Returns a search pipeline",
-      "url": "https://opensearch.org/docs/latest/opensearch/rest-api/search_pipelines/"
+      "url": "https://opensearch.org/docs/latest/search-plugins/search-pipelines/retrieving-search-pipeline/"
     },
     "stability": "stable",
     "url": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search_pipeline.put.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search_pipeline.put.json
index b7375d36825a2..90d4baea4ecf2 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/search_pipeline.put.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search_pipeline.put.json
@@ -2,7 +2,7 @@
   "search_pipeline.put": {
     "documentation": {
       "description": "Creates or updates a search pipeline.",
-      "url": "https://opensearch.org/docs/latest/opensearch/rest-api/search_pipelines/"
+      "url": "https://opensearch.org/docs/latest/search-plugins/search-pipelines/creating-search-pipeline/"
     },
     "stability": "stable",
     "url": {
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json
index 4230b660523b8..00fa06bb96c7e 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json
@@ -1,7 +1,7 @@
 {
   "search_template":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/current/search-template.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/search-template/",
       "description":"Allows to use the Mustache language to pre-render a search definition."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.cleanup_repository.json b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.cleanup_repository.json
index 05eb3309b11e6..826660ce58dea 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.cleanup_repository.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.cleanup_repository.json
@@ -1,7 +1,7 @@
 {
   "snapshot.cleanup_repository": {
     "documentation": {
-      "url": "https://www.elastic.co/guide/en/elasticsearch/reference/master/clean-up-snapshot-repo-api.html",
+      "url": "https://opensearch.org/docs/latest/api-reference/snapshots/index/",
       "description": "Removes stale data from repository."
     },
     "stability": "stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.clone.json b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.clone.json
index c79460fc30a48..ae83d9fa6497a 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.clone.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.clone.json
@@ -1,7 +1,7 @@
 {
   "snapshot.clone":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/modules-snapshots.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/snapshots/index/",
       "description":"Clones indices from one snapshot into another snapshot in the same repository."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.create.json b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.create.json
index 64aaeaef9d897..9b4abd1b41a93 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.create.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.create.json
@@ -1,7 +1,7 @@
 {
   "snapshot.create":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/modules-snapshots.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/snapshots/create-snapshot/",
       "description":"Creates a snapshot in a repository."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.create_repository.json b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.create_repository.json
index 4965162bcd86c..02f7350495cfc 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.create_repository.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.create_repository.json
@@ -1,7 +1,7 @@
 {
   "snapshot.create_repository":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/modules-snapshots.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/snapshots/create-repository/",
       "description":"Creates a repository."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.delete.json b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.delete.json
index 2e21a08219942..205872dfa95ea 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.delete.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.delete.json
@@ -1,7 +1,7 @@
 {
   "snapshot.delete":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/modules-snapshots.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/snapshots/delete-snapshot/",
       "description":"Deletes a snapshot."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.delete_repository.json b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.delete_repository.json
index 3fc22f969784c..61a8c2b5f8086 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.delete_repository.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.delete_repository.json
@@ -1,7 +1,7 @@
 {
   "snapshot.delete_repository":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/modules-snapshots.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/snapshots/delete-snapshot-repository/",
       "description":"Deletes a repository."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.get.json b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.get.json
index e084a997a61b1..41b7d728da63f 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.get.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.get.json
@@ -1,7 +1,7 @@
 {
   "snapshot.get":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/modules-snapshots.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/snapshots/get-snapshot/",
       "description":"Returns information about a snapshot."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.get_repository.json b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.get_repository.json
index cf03bab18c03f..dc3e4a91d2a77 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.get_repository.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.get_repository.json
@@ -1,7 +1,7 @@
 {
   "snapshot.get_repository":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/modules-snapshots.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/snapshots/get-snapshot-repository/",
       "description":"Returns information about a repository."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.restore.json b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.restore.json
index 07148c7d261f4..401f612a33203 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.restore.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.restore.json
@@ -1,7 +1,7 @@
 {
   "snapshot.restore":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/modules-snapshots.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/snapshots/restore-snapshot/",
       "description":"Restores a snapshot."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.status.json b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.status.json
index 4f22c24fd9a56..1ac6042941013 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.status.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.status.json
@@ -1,7 +1,7 @@
 {
   "snapshot.status":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/modules-snapshots.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/snapshots/get-snapshot-status/",
       "description":"Returns information about the status of a snapshot."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.verify_repository.json b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.verify_repository.json
index 865eb15d11310..fc0fd400f8bbd 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.verify_repository.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/snapshot.verify_repository.json
@@ -1,7 +1,7 @@
 {
   "snapshot.verify_repository":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/modules-snapshots.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/snapshots/verify-snapshot-repository/",
       "description":"Verifies a repository."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.cancel.json b/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.cancel.json
index 32f90abab60f6..70dc99d7eb2da 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.cancel.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.cancel.json
@@ -1,7 +1,7 @@
 {
   "tasks.cancel":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/tasks.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/tasks/",
       "description":"Cancels a task, if it can be cancelled through an API."
     },
     "stability":"experimental",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.get.json b/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.get.json
index 63646ae539de5..d10fe62aa84b8 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.get.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.get.json
@@ -1,7 +1,7 @@
 {
   "tasks.get":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/tasks.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/tasks/",
       "description":"Returns information about a task."
     },
     "stability":"experimental",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.list.json b/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.list.json
index 7137114c96bff..20a6cb5f7f7d6 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.list.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/tasks.list.json
@@ -1,7 +1,7 @@
 {
   "tasks.list":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/tasks.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/tasks/",
       "description":"Returns a list of tasks."
     },
     "stability":"experimental",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/update.json b/rest-api-spec/src/main/resources/rest-api-spec/api/update.json
index c8d1ed435756b..131036486e024 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/update.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/update.json
@@ -1,7 +1,7 @@
 {
   "update":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/docs-update.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/document-apis/update-document/",
       "description":"Updates a document with a script or partial document."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/update_by_query.json b/rest-api-spec/src/main/resources/rest-api-spec/api/update_by_query.json
index 71a0c1fc8ad95..7f6cb797da861 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/update_by_query.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/update_by_query.json
@@ -1,7 +1,7 @@
 {
   "update_by_query":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/master/docs-update-by-query.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/document-apis/update-by-query/",
       "description":"Performs an update on every document in the index without changing the source,\nfor example to pick up a mapping change."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/update_by_query_rethrottle.json b/rest-api-spec/src/main/resources/rest-api-spec/api/update_by_query_rethrottle.json
index bd70f6e1231c9..bc266ed7df68f 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/api/update_by_query_rethrottle.json
+++ b/rest-api-spec/src/main/resources/rest-api-spec/api/update_by_query_rethrottle.json
@@ -1,7 +1,7 @@
 {
   "update_by_query_rethrottle":{
     "documentation":{
-      "url":"https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-update-by-query.html",
+      "url":"https://opensearch.org/docs/latest/api-reference/document-apis/update-by-query/",
       "description":"Changes the number of requests per second for a particular Update By Query operation."
     },
     "stability":"stable",
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml
index 29fbf55417961..989ea6b93f47f 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/cat.shards/10_basic.yml
@@ -1,13 +1,108 @@
 "Help":
   - skip:
-      version: " - 2.11.99"
+      version: " - 2.13.99"
+      reason:  search idle reactivate count total is only added in 3.0.0
+      features: node_selector
+  - do:
+      cat.shards:
+        help: true
+      node_selector:
+        version: "2.14.0 - "
+
+  - match:
+      $body: |
+        /^   index                            .+   \n
+             shard                            .+   \n
+             prirep                           .+   \n
+             state                            .+   \n
+             docs                             .+   \n
+             store                            .+   \n
+             ip                               .+   \n
+             id                               .+   \n
+             node                             .+   \n
+             sync_id                          .+   \n
+             unassigned.reason                .+   \n
+             unassigned.at                    .+   \n
+             unassigned.for                   .+   \n
+             unassigned.details               .+   \n
+             recoverysource.type              .+   \n
+             completion.size                  .+   \n
+             fielddata.memory_size            .+   \n
+             fielddata.evictions              .+   \n
+             query_cache.memory_size          .+   \n
+             query_cache.evictions            .+   \n
+             flush.total                      .+   \n
+             flush.total_time                 .+   \n
+             get.current                      .+   \n
+             get.time                         .+   \n
+             get.total                        .+   \n
+             get.exists_time                  .+   \n
+             get.exists_total                 .+   \n
+             get.missing_time                 .+   \n
+             get.missing_total                .+   \n
+             indexing.delete_current          .+   \n
+             indexing.delete_time             .+   \n
+             indexing.delete_total            .+   \n
+             indexing.index_current           .+   \n
+             indexing.index_time              .+   \n
+             indexing.index_total             .+   \n
+             indexing.index_failed            .+   \n
+             merges.current                   .+   \n
+             merges.current_docs              .+   \n
+             merges.current_size              .+   \n
+             merges.total                     .+   \n
+             merges.total_docs                .+   \n
+             merges.total_size                .+   \n
+             merges.total_time                .+   \n
+             refresh.total                    .+   \n
+             refresh.time                     .+   \n
+             refresh.external_total           .+   \n
+             refresh.external_time            .+   \n
+             refresh.listeners                .+   \n
+             search.fetch_current             .+   \n
+             search.fetch_time                .+   \n
+             search.fetch_total               .+   \n
+             search.open_contexts             .+   \n
+             search.query_current             .+   \n
+             search.query_time                .+   \n
+             search.query_total               .+   \n
+             search.concurrent_query_current  .+   \n
+             search.concurrent_query_time     .+   \n
+             search.concurrent_query_total    .+   \n
+             search.concurrent_avg_slice_count  .+   \n
+             search.scroll_current            .+   \n
+             search.scroll_time               .+   \n
+             search.scroll_total              .+   \n
+             search.point_in_time_current     .+   \n
+             search.point_in_time_time        .+   \n
+             search.point_in_time_total       .+   \n
+             search.search_idle_reactivate_count_total  .+   \n
+             segments.count                   .+   \n
+             segments.memory                  .+   \n
+             segments.index_writer_memory     .+   \n
+             segments.version_map_memory      .+   \n
+             segments.fixed_bitset_memory     .+   \n
+             seq_no.max                       .+   \n
+             seq_no.local_checkpoint          .+   \n
+             seq_no.global_checkpoint         .+   \n
+             warmer.current                   .+   \n
+             warmer.total                     .+   \n
+             warmer.total_time                .+   \n
+             path.data                        .+   \n
+             path.state                       .+   \n
+             docs.deleted                     .+   \n
+        $/
+---
+"Help from 2.12.0 to 2.13.99":
+  - skip:
+      version: " - 2.11.99 , 2.14.0 - "
       reason:  deleted docs and concurrent search are added in 2.12.0
       features: node_selector
   - do:
       cat.shards:
         help: true
       node_selector:
-        version: "2.12.0 - "
+        version: "2.12.0 - 2.13.99"
 
   - match:
       $body: |
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.clone/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.clone/10_basic.yml
index fa48820a71a89..07df09225c624 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.clone/10_basic.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.clone/10_basic.yml
@@ -113,12 +113,12 @@ setup:
             index.number_of_replicas: 0
             index.number_of_shards: 6
 
+# Related issue: https://github.com/opensearch-project/OpenSearch/issues/4845
 ---
 "Returns error if target index's metadata write is blocked":
-
   - skip:
-      version: " - 2.99.99"
-      reason: "only available in 3.0+"
+      version: " - 2.7.99"
+      reason: "the bug was fixed in 2.8.0"
 
   # block source index's write operations
   - do:
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.forcemerge/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.forcemerge/10_basic.yml
index 39fb1604d9596..7410e020e1a91 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.forcemerge/10_basic.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.forcemerge/10_basic.yml
@@ -31,8 +31,8 @@
 ---
 "Test primary_only parameter":
   - skip:
-      version: " - 2.99.99"
-      reason: "primary_only is available in 3.0+"
+      version: " - 2.12.99"
+      reason: "primary_only is available in 2.13.0+"
 
   - do:
       indices.create:
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/10_basic.yml
index 426729e737978..67b5be7eb0fd5 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/10_basic.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.shrink/10_basic.yml
@@ -72,12 +72,12 @@ setup:
   - match: { _id:      "1"     }
   - match: { _source:  { foo: "hello world" } }
 
+# Related issue: https://github.com/opensearch-project/OpenSearch/issues/4845
 ---
 "Returns error if target index's metadata write is blocked":
-
   - skip:
-      version: " - 2.99.99"
-      reason: "only available in 3.0+"
+      version: " - 2.7.99"
+      reason: "the bug was fixed in 2.8.0"
 
   # block source index's write operations
   - do:
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml
index 50c2819eac9d5..096a61a765288 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.split/10_basic.yml
@@ -219,12 +219,12 @@ setup:
             index.number_of_replicas: 0
             index.number_of_shards: 6
 
+# Related issue: https://github.com/opensearch-project/OpenSearch/issues/4845
 ---
 "Returns error if target index's metadata write is blocked":
-
   - skip:
-      version: " - 2.99.99"
-      reason: "only available in 3.0+"
+      version: " - 2.7.99"
+      reason: "the bug was fixed in 2.8.0"
 
   # block source index's write operations
   - do:
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/10_histogram.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/10_histogram.yml
index e7da9a0bc454c..fa71137912a91 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/10_histogram.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/10_histogram.yml
@@ -644,3 +644,59 @@ setup:
   - match: { aggregations.histo.buckets.0.doc_count: 1 }
   - match: { aggregations.histo.buckets.20.key: 20 }
   - match: { aggregations.histo.buckets.20.doc_count: 1 }
+
+---
+"date_histogram profiler shows filter rewrite info":
+  - skip:
+      version: " - 2.99.99"
+      reason:  debug info for filter rewrite added in 3.0.0 (to be backported to 2.14.0)
+
+  - do:
+      indices.create:
+        index: test_2
+        body:
+          settings:
+            number_of_replicas: 0
+            number_of_shards: 1
+          mappings:
+            properties:
+              date:
+                type: date
+
+  - do:
+      bulk:
+        index: test_2
+        refresh: true
+        body:
+          - '{"index": {}}'
+          - '{"date": "2016-01-01"}'
+          - '{"index": {}}'
+          - '{"date": "2016-01-02"}'
+          - '{"index": {}}'
+          - '{"date": "2016-02-01"}'
+          - '{"index": {}}'
+          - '{"date": "2016-03-01"}'
+
+  - do:
+      search:
+        index: test_2
+        body:
+          size: 0
+          profile: true
+          aggs:
+            histo:
+              date_histogram:
+                field: date
+                calendar_interval: month
+
+  - match: { hits.total.value: 4 }
+  - length: { aggregations.histo.buckets: 3 }
+  - match: { aggregations.histo.buckets.0.key_as_string: "2016-01-01T00:00:00.000Z" }
+  - match: { aggregations.histo.buckets.0.doc_count: 2 }
+  - match: { profile.shards.0.aggregations.0.type: DateHistogramAggregator }
+  - match: { profile.shards.0.aggregations.0.description: histo }
+  - match: { profile.shards.0.aggregations.0.debug.total_buckets: 3 }
+  - match: { profile.shards.0.aggregations.0.debug.optimized_segments: 1 }
+  - match: { profile.shards.0.aggregations.0.debug.unoptimized_segments: 0 }
+  - match: { profile.shards.0.aggregations.0.debug.leaf_visited: 1 }
+  - match: { profile.shards.0.aggregations.0.debug.inner_visited: 0 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml
index 2808be8cd7045..3a0099dae3b33 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml
@@ -1069,3 +1069,61 @@ setup:
   - match: { aggregations.test.buckets.1.doc_count: 2 }
   - match: { aggregations.test.buckets.2.key.kw: null }
   - match: { aggregations.test.buckets.2.doc_count: 2 }
+
+---
+"composite aggregation date_histogram profile shows filter rewrite info":
+  - skip:
+      version: " - 2.99.99"
+      reason:  debug info for filter rewrite added in 3.0.0 (to be backported to 2.14.0)
+
+  - do:
+      indices.create:
+        index: test_2
+        body:
+          settings:
+            number_of_replicas: 0
+            number_of_shards: 1
+          mappings:
+            properties:
+              date:
+                type: date
+  - do:
+      bulk:
+        index: test_2
+        refresh: true
+        body:
+          - '{"index": {}}'
+          - '{"date": "2016-01-01"}'
+          - '{"index": {}}'
+          - '{"date": "2016-01-02"}'
+          - '{"index": {}}'
+          - '{"date": "2016-02-01"}'
+          - '{"index": {}}'
+          - '{"date": "2016-03-01"}'
+  - do:
+      search:
+        index: test_2
+        body:
+          size: 0
+          profile: true
+          aggregations:
+            test:
+              composite:
+                sources: [
+                  {
+                    "date": {
+                      "date_histogram": {
+                        "field": "date",
+                        "calendar_interval": "1d",
+                        "format": "strict_date"
+                      }
+                    }
+                  }
+                ]
+
+  - match: { hits.total.value: 4 }
+  - length: { aggregations.test.buckets: 4 }
+  - match: { profile.shards.0.aggregations.0.debug.optimized_segments: 1 }
+  - match: { profile.shards.0.aggregations.0.debug.unoptimized_segments: 0 }
+  - match: { profile.shards.0.aggregations.0.debug.leaf_visited: 1 }
+  - match: { profile.shards.0.aggregations.0.debug.inner_visited: 0 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/330_auto_date_histogram.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/330_auto_date_histogram.yml
index 6b5e06a549be3..1356eac41ae79 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/330_auto_date_histogram.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/330_auto_date_histogram.yml
@@ -99,3 +99,29 @@ setup:
   - length: { aggregations.histo.buckets: 2 }
   - match: { profile.shards.0.aggregations.0.type: AutoDateHistogramAggregator.FromSingle }
   - match: { profile.shards.0.aggregations.0.debug.surviving_buckets: 4 }
+
+---
+"auto_date_histogram profile shows filter rewrite info":
+  - skip:
+      version: " - 2.99.99"
+      reason: debug info for filter rewrite added in 3.0.0 (to be backported to 2.14.0)
+
+  - do:
+      search:
+        body:
+          profile: true
+          size: 0
+          aggs:
+            histo:
+              auto_date_histogram:
+                field: date
+                buckets: 2
+
+  - match: { hits.total.value: 4 }
+  - length: { aggregations.histo.buckets: 2 }
+  - match: { profile.shards.0.aggregations.0.type: AutoDateHistogramAggregator.FromSingle }
+  - match: { profile.shards.0.aggregations.0.debug.surviving_buckets: 4 }
+  - match: { profile.shards.0.aggregations.0.debug.optimized_segments: 1 }
+  - match: { profile.shards.0.aggregations.0.debug.unoptimized_segments: 0 }
+  - match: { profile.shards.0.aggregations.0.debug.leaf_visited: 1 }
+  - match: { profile.shards.0.aggregations.0.debug.inner_visited: 0 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/370_multi_terms.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/370_multi_terms.yml
index 7db5f31d8e761..f9354db46a384 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/370_multi_terms.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/370_multi_terms.yml
@@ -760,3 +760,138 @@ setup:
   - match: { aggregations.histo.buckets.2.key_as_string: "2022-03-25T00:00:00.000Z" }
   - match: { aggregations.histo.buckets.2.m_terms.buckets.0.key: [ "a", "127.0.0.1" ] }
   - match: { aggregations.histo.buckets.2.m_terms.buckets.1.key: [ "b", "127.0.0.1" ] }
+
+---
+"multi_terms bucket and multiple orders test":
+  - skip:
+      version: "- 2.0.99"
+      reason:  multi_terms aggregation is introduced in 2.1.0
+
+  - do:
+      bulk:
+        index: test_1
+        refresh: true
+        body:
+          - '{"index": {}}'
+          - '{"str": "a", "integer": 1, "double": 1234.5, "boolean": true}'
+          - '{"index": {}}'
+          - '{"str": "a", "integer": 2, "double": 5678.9, "boolean": true}'
+          - '{"index": {}}'
+          - '{"str": "a", "integer": 1, "double": 123.4, "boolean": false}'
+          - '{"index": {}}'
+          - '{"str": "a", "integer": 2, "double": 456.7, "boolean": false}'
+          - '{"index": {}}'
+          - '{"str": "b", "integer": 3, "double": 1234.5, "boolean": true}'
+          - '{"index": {}}'
+          - '{"str": "b", "integer": 4, "double": 5678.9, "boolean": true}'
+          - '{"index": {}}'
+          - '{"str": "b", "integer": 3, "double": 234.5, "boolean": false}'
+          - '{"index": {}}'
+          - '{"str": "b", "integer": 4, "double": 456.7, "boolean": false}'
+
+  - do:
+      search:
+        index: test_1
+        size: 0
+        body:
+          aggs:
+            m_terms:
+              multi_terms:
+                terms:
+                  - field: str
+                  - field: boolean
+                order:
+                  - max_int: desc
+                  - min_double: asc
+              aggs:
+                max_int:
+                  max:
+                    field: integer
+                min_double:
+                  min:
+                    field: double
+
+  - length: { aggregations.m_terms.buckets: 4 }
+  - match: { aggregations.m_terms.buckets.0.key: ["b", false] }
+  - match: { aggregations.m_terms.buckets.0.key_as_string: "b|false" }
+  - match: { aggregations.m_terms.buckets.0.doc_count: 2 }
+  - match: { aggregations.m_terms.buckets.0.max_int.value: 4.0 }
+  - match: { aggregations.m_terms.buckets.0.min_double.value: 234.5 }
+  - match: { aggregations.m_terms.buckets.1.key: ["b", true] }
+  - match: { aggregations.m_terms.buckets.1.key_as_string: "b|true" }
+  - match: { aggregations.m_terms.buckets.1.doc_count: 2 }
+  - match: { aggregations.m_terms.buckets.1.max_int.value: 4.0 }
+  - match: { aggregations.m_terms.buckets.1.min_double.value: 1234.5 }
+  - match: { aggregations.m_terms.buckets.2.key: [ "a", false ] }
+  - match: { aggregations.m_terms.buckets.2.key_as_string: "a|false" }
+  - match: { aggregations.m_terms.buckets.2.doc_count: 2 }
+  - match: { aggregations.m_terms.buckets.2.max_int.value: 2.0 }
+  - match: { aggregations.m_terms.buckets.2.min_double.value: 123.4 }
+  - match: { aggregations.m_terms.buckets.3.key: [ "a", true ] }
+  - match: { aggregations.m_terms.buckets.3.key_as_string: "a|true" }
+  - match: { aggregations.m_terms.buckets.3.doc_count: 2 }
+  - match: { aggregations.m_terms.buckets.3.max_int.value: 2.0 }
+  - match: { aggregations.m_terms.buckets.3.min_double.value: 1234.5 }
+
+---
+"multi_terms bucket and multiple orders with size test":
+  - skip:
+      version: "- 2.0.99"
+      reason:  multi_terms aggregation is introduced in 2.1.0
+
+  - do:
+      bulk:
+        index: test_1
+        refresh: true
+        body:
+          - '{"index": {}}'
+          - '{"str": "a", "integer": 1, "double": 1234.5, "boolean": true}'
+          - '{"index": {}}'
+          - '{"str": "a", "integer": 2, "double": 5678.9, "boolean": true}'
+          - '{"index": {}}'
+          - '{"str": "a", "integer": 1, "double": 123.4, "boolean": false}'
+          - '{"index": {}}'
+          - '{"str": "a", "integer": 2, "double": 456.7, "boolean": false}'
+          - '{"index": {}}'
+          - '{"str": "b", "integer": 3, "double": 1234.5, "boolean": true}'
+          - '{"index": {}}'
+          - '{"str": "b", "integer": 4, "double": 5678.9, "boolean": true}'
+          - '{"index": {}}'
+          - '{"str": "b", "integer": 3, "double": 234.5, "boolean": false}'
+          - '{"index": {}}'
+          - '{"str": "b", "integer": 4, "double": 456.7, "boolean": false}'
+
+  - do:
+      search:
+        index: test_1
+        size: 0
+        body:
+          aggs:
+            m_terms:
+              multi_terms:
+                terms:
+                  - field: str
+                  - field: boolean
+                order:
+                  - max_int: desc
+                  - min_double: asc
+                size: 2
+              aggs:
+                max_int:
+                  max:
+                    field: integer
+                min_double:
+                  min:
+                    field: double
+
+  - length: { aggregations.m_terms.buckets: 2 }
+  - match: { aggregations.m_terms.buckets.0.key: ["b", false] }
+  - match: { aggregations.m_terms.buckets.0.key_as_string: "b|false" }
+  - match: { aggregations.m_terms.buckets.0.doc_count: 2 }
+  - match: { aggregations.m_terms.buckets.0.max_int.value: 4.0 }
+  - match: { aggregations.m_terms.buckets.0.min_double.value: 234.5 }
+  - match: { aggregations.m_terms.buckets.1.key: ["b", true] }
+  - match: { aggregations.m_terms.buckets.1.key_as_string: "b|true" }
+  - match: { aggregations.m_terms.buckets.1.doc_count: 2 }
+  - match: { aggregations.m_terms.buckets.1.max_int.value: 4.0 }
+  - match: { aggregations.m_terms.buckets.1.min_double.value: 1234.5 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/400_inner_hits.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/400_inner_hits.yml
new file mode 100644
index 0000000000000..cbfc92a40835c
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/400_inner_hits.yml
@@ -0,0 +1,68 @@
+setup:
+  - do:
+      indices.create:
+          index: test_1
+          body:
+            settings:
+              number_of_replicas: 0
+            mappings:
+              properties:
+                list_id:
+                  type: integer
+                names:
+                  type: nested
+                  properties:
+                    full_name:
+                      type: text
+
+  - do:
+       bulk:
+         refresh: true
+         body:
+           - index:
+               _index: test_1
+               _id:    1
+           - list_id: 1
+             names:
+               - full_name: John Doe
+               - full_name: John Micheal Doe
+           - index:
+               _index: test_1
+               _id:    2
+           - list_id: 2
+             names:
+               - full_name: Jane Doe
+               - full_name: Jane Michelle Doe
+
+---
+"Include inner hits in top hits":
+  - skip:
+      version: " - 2.13.99"
+      reason: "the regression was fixed in 2.14.0"
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        body:
+          query:
+            nested:
+              path: names
+              query:
+                match:
+                  names.full_name: Doe
+              inner_hits: { }
+          size: 0
+          aggs:
+            lists:
+              terms:
+                field: list_id
+              aggs:
+                top_result:
+                  top_hits:
+                    size: 10
+
+  - length: { hits.hits: 0 }
+  - length: { aggregations.lists.buckets: 2 }
+  - length: { aggregations.lists.buckets.0.top_result.hits.hits: 1 }
+  - length: { aggregations.lists.buckets.0.top_result.hits.hits.0.inner_hits.names.hits.hits: 2 }
+  - length: { aggregations.lists.buckets.1.top_result.hits.hits: 1 }
+  - length: { aggregations.lists.buckets.1.top_result.hits.hits.0.inner_hits.names.hits.hits: 2 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/340_doc_values_field.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/340_doc_values_field.yml
index d5ece1719dc48..a133060f07c6f 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/340_doc_values_field.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/340_doc_values_field.yml
@@ -46,6 +46,18 @@
                 type: ip
                 index: true
                 doc_values: true
+              boolean:
+                  type: boolean
+                  index: true
+                  doc_values: true
+              date:
+                type: date
+                index: true
+                doc_values: true
+              date_nanos:
+                type: date_nanos
+                index: true
+                doc_values: true
 
   - do:
       bulk:
@@ -53,11 +65,11 @@
         refresh: true
         body:
           - '{"index": {"_index": "test-iodvq", "_id": "1" }}'
-          - '{ "some_keyword": "ingesting some random keyword data", "byte": 120, "double": 100.0, "float": "800.0", "half_float": "400.0", "integer": 1290, "long": 13456, "short": 150, "unsigned_long": 10223372036854775800, "ip_field": "192.168.0.1" }'
+          - '{ "some_keyword": "ingesting some random keyword data", "byte": 120, "double": 100.0, "float": "800.0", "half_float": "400.0", "integer": 1290, "long": 13456, "short": 150, "unsigned_long": 10223372036854775800, "ip_field": "192.168.0.1", "boolean": true, "date_nanos": "2018-10-29T12:12:12.123456789Z", "date": "2018-10-29T12:12:12.987Z" }'
           - '{ "index": { "_index": "test-iodvq", "_id": "2" }}'
-          - '{ "some_keyword": "400", "byte": 121, "double": 101.0, "float": "801.0", "half_float": "401.0", "integer": 1291, "long": 13457, "short": 151, "unsigned_long": 10223372036854775801, "ip_field": "192.168.0.2" }'
+          - '{ "some_keyword": "400", "byte": 121, "double": 101.0, "float": "801.0", "half_float": "401.0", "integer": 1291, "long": 13457, "short": 151, "unsigned_long": 10223372036854775801, "ip_field": "192.168.0.2", "boolean": true, "date_nanos": "2020-10-29T12:12:12.987654321Z", "date": "2020-10-29T12:12:12.987Z" }'
           - '{ "index": { "_index": "test-iodvq", "_id": "3" } }'
-          - '{ "some_keyword": "5", "byte": 122, "double": 102.0, "float": "802.0", "half_float": "402.0", "integer": 1292, "long": 13458, "short": 152, "unsigned_long": 10223372036854775802, "ip_field": "192.168.0.3" }'
+          - '{ "some_keyword": "5", "byte": 122, "double": 102.0, "float": "802.0", "half_float": "402.0", "integer": 1292, "long": 13458, "short": 152, "unsigned_long": 10223372036854775802, "ip_field": "192.168.0.3", "boolean": false, "date_nanos": "2024-10-29T12:12:12.987654321Z", "date": "2024-10-29T12:12:12.987Z" }'
 
   - do:
       search:
@@ -183,6 +195,17 @@
 
   - match: {hits.total: 1}
 
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-iodvq
+        body:
+          query:
+            term:
+              boolean: true
+
+  - match: { hits.total: 2 }
+
   - do:
       search:
         rest_total_hits_as_int: true
@@ -282,6 +305,17 @@
 
   - match: { hits.total: 2 }
 
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-iodvq
+        body:
+          query:
+            terms:
+              boolean: [true, false]
+
+  - match: { hits.total: 3 }
+
   - do:
       search:
         rest_total_hits_as_int: true
@@ -415,6 +449,92 @@
 
   - match: { hits.total: 2 }
 
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-iodvq
+        body:
+          query:
+            range: {
+              date_nanos: {
+                gte: "2018-10-29T12:12:12.123456789Z"
+              },
+            }
+
+  - match: { hits.total: 3 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-iodvq
+        body:
+          query:
+            range: {
+              date: {
+                gte: "2018-10-29T12:12:12.987Z",
+                lte: "2020-10-29T12:12:12.987Z"
+              },
+            }
+
+  - match: { hits.total: 2 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-iodvq
+        body:
+          query:
+            range: {
+              boolean: {
+                gte: true
+              },
+            }
+
+  - match: { hits.total: 2 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-iodvq
+        body:
+          query:
+            range: {
+              boolean: {
+                lte: true
+              },
+            }
+
+  - match: { hits.total: 3 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-iodvq
+        body:
+          query:
+            range: {
+              boolean: {
+                lte: true,
+                gte: false
+              },
+            }
+
+  - match: { hits.total: 3 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-iodvq
+        body:
+          query:
+            range: {
+              boolean: {
+                lte: false,
+                gte: true
+              },
+            }
+
+  - match: { hits.total: 0 }
 ---
 "search on fields with only index enabled":
   - do:
@@ -463,6 +583,18 @@
                 type: ip
                 index: true
                 doc_values: false
+              boolean:
+                type: boolean
+                index: true
+                doc_values: false
+              date_nanos:
+                type: date_nanos
+                index: true
+                doc_values: false
+              date:
+                type: date
+                index: true
+                doc_values: false
 
   - do:
       bulk:
@@ -470,11 +602,11 @@
         refresh: true
         body:
           - '{"index": {"_index": "test-index", "_id": "1" }}'
-          - '{ "some_keyword": "ingesting some random keyword data", "byte": 120, "double": 100.0, "float": "800.0", "half_float": "400.0", "integer": 1290, "long": 13456, "short": 150, "unsigned_long": 10223372036854775800, "ip_field": "192.168.0.1" }'
+          - '{ "some_keyword": "ingesting some random keyword data", "byte": 120, "double": 100.0, "float": "800.0", "half_float": "400.0", "integer": 1290, "long": 13456, "short": 150, "unsigned_long": 10223372036854775800, "ip_field": "192.168.0.1", "boolean": true, "date_nanos": "2018-10-29T12:12:12.123456789Z", "date": "2018-10-29T12:12:12.987Z" }'
           - '{ "index": { "_index": "test-index", "_id": "2" }}'
-          - '{ "some_keyword": "400", "byte": 121, "double": 101.0, "float": "801.0", "half_float": "401.0", "integer": 1291, "long": 13457, "short": 151, "unsigned_long": 10223372036854775801, "ip_field": "192.168.0.2" }'
+          - '{ "some_keyword": "400", "byte": 121, "double": 101.0, "float": "801.0", "half_float": "401.0", "integer": 1291, "long": 13457, "short": 151, "unsigned_long": 10223372036854775801, "ip_field": "192.168.0.2", "boolean": true, "date_nanos": "2020-10-29T12:12:12.123456789Z", "date": "2020-10-29T12:12:12.987Z" }'
           - '{ "index": { "_index": "test-index", "_id": "3" } }'
-          - '{ "some_keyword": "5", "byte": 122, "double": 102.0, "float": "802.0", "half_float": "402.0", "integer": 1292, "long": 13458, "short": 152, "unsigned_long": 10223372036854775802, "ip_field": "192.168.0.3" }'
+          - '{ "some_keyword": "5", "byte": 122, "double": 102.0, "float": "802.0", "half_float": "402.0", "integer": 1292, "long": 13458, "short": 152, "unsigned_long": 10223372036854775802, "ip_field": "192.168.0.3", "boolean": false, "date_nanos": "2024-10-29T12:12:12.123456789Z", "date": "2024-10-29T12:12:12.987Z" }'
 
   - do:
       search:
@@ -600,6 +732,18 @@
 
   - match: {hits.total: 1}
 
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-index
+        body:
+          query:
+            term:
+              boolean: true
+
+  - match: { hits.total: 2 }
+
+
   - do:
       search:
         rest_total_hits_as_int: true
@@ -699,6 +843,17 @@
 
   - match: { hits.total: 2 }
 
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-index
+        body:
+          query:
+            terms:
+              boolean: [true, false]
+
+  - match: { hits.total: 3 }
+
   - do:
       search:
         rest_total_hits_as_int: true
@@ -831,6 +986,93 @@
                 lte: "192.168.0.2"
 
   - match: { hits.total: 2 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-index
+        body:
+          query:
+            range: {
+              date_nanos: {
+                gte: "2018-10-29T12:12:12.123456789Z"
+              },
+            }
+
+  - match: { hits.total: 3 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-index
+        body:
+          query:
+            range: {
+              date: {
+                gte: "2018-10-29T12:12:12.987Z",
+                lte: "2020-10-29T12:12:12.987Z"
+              },
+            }
+
+  - match: { hits.total: 2 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-index
+        body:
+          query:
+            range: {
+              boolean: {
+                gte: true
+              },
+            }
+
+  - match: { hits.total: 2 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-index
+        body:
+          query:
+            range: {
+              boolean: {
+                lte: true
+              },
+            }
+
+  - match: { hits.total: 3 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-index
+        body:
+          query:
+            range: {
+              boolean: {
+                lte: true,
+                gte: false
+              },
+            }
+
+  - match: { hits.total: 3 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-index
+        body:
+          query:
+            range: {
+              boolean: {
+                lte: false,
+                gte: true
+              },
+            }
+
+  - match: { hits.total: 0 }
 ---
 "search on fields with only doc_values enabled":
   - skip:
@@ -883,6 +1125,18 @@
                 type: ip
                 index: false
                 doc_values: true
+              boolean:
+                type: boolean
+                index: false
+                doc_values: true
+              date_nanos:
+                type: date_nanos
+                index: false
+                doc_values: true
+              date:
+                type: date
+                index: false
+                doc_values: true
 
   - do:
       bulk:
@@ -890,11 +1144,11 @@
         refresh: true
         body:
           - '{"index": {"_index": "test-doc-values", "_id": "1" }}'
-          - '{ "some_keyword": "ingesting some random keyword data", "byte": 120, "double": 100.0, "float": "800.0", "half_float": "400.0", "integer": 1290, "long": 13456, "short": 150, "unsigned_long": 10223372036854775800, "ip_field": "192.168.0.1" }'
+          - '{ "some_keyword": "ingesting some random keyword data", "byte": 120, "double": 100.0, "float": "800.0", "half_float": "400.0", "integer": 1290, "long": 13456, "short": 150, "unsigned_long": 10223372036854775800, "ip_field": "192.168.0.1", "boolean": true, "date_nanos": "2018-10-29T12:12:12.123456789Z", "date": "2018-10-29T12:12:12.987Z" }'
           - '{ "index": { "_index": "test-doc-values", "_id": "2" }}'
-          - '{ "some_keyword": "400", "byte": 121, "double": 101.0, "float": "801.0", "half_float": "401.0", "integer": 1291, "long": 13457, "short": 151, "unsigned_long": 10223372036854775801, "ip_field": "192.168.0.2" }'
+          - '{ "some_keyword": "400", "byte": 121, "double": 101.0, "float": "801.0", "half_float": "401.0", "integer": 1291, "long": 13457, "short": 151, "unsigned_long": 10223372036854775801, "ip_field": "192.168.0.2", "boolean": true, "date_nanos": "2020-10-29T12:12:12.123456789Z", "date": "2020-10-29T12:12:12.987Z" }'
           - '{ "index": { "_index": "test-doc-values", "_id": "3" } }'
-          - '{ "some_keyword": "5", "byte": 122, "double": 102.0, "float": "802.0", "half_float": "402.0", "integer": 1292, "long": 13458, "short": 152, "unsigned_long": 10223372036854775802, "ip_field": "192.168.0.3" }'
+          - '{ "some_keyword": "5", "byte": 122, "double": 102.0, "float": "802.0", "half_float": "402.0", "integer": 1292, "long": 13458, "short": 152, "unsigned_long": 10223372036854775802, "ip_field": "192.168.0.3", "boolean": false, "date_nanos": "2024-10-29T12:12:12.123456789Z", "date": "2024-10-29T12:12:12.987Z" }'
 
   - do:
       search:
@@ -1019,6 +1273,17 @@
 
   - match: { hits.total: 1 }
 
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-doc-values
+        body:
+          query:
+            term:
+              boolean: false
+
+  - match: { hits.total: 1 }
+
   - do:
       search:
         rest_total_hits_as_int: true
@@ -1107,6 +1372,17 @@
 
   - match: { hits.total: 2 }
 
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-doc-values
+        body:
+          query:
+            terms:
+              boolean: [true, false]
+
+  - match: { hits.total: 3 }
+
   - do:
       search:
         rest_total_hits_as_int: true
@@ -1239,3 +1515,90 @@
                 lte: "192.168.0.2"
 
   - match: { hits.total: 2 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-doc-values
+        body:
+          query:
+            range: {
+              date_nanos: {
+                gte: "2018-10-29T12:12:12.123456789Z"
+              },
+            }
+
+  - match: { hits.total: 3 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-doc-values
+        body:
+          query:
+            range: {
+              date: {
+                gte: "2018-10-29T12:12:12.987Z",
+                lte: "2020-10-29T12:12:12.987Z"
+              },
+            }
+
+  - match: { hits.total: 2 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-doc-values
+        body:
+          query:
+            range: {
+              boolean: {
+                gte: true
+              },
+            }
+
+  - match: { hits.total: 2 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-doc-values
+        body:
+          query:
+            range: {
+              boolean: {
+                lte: true
+              },
+            }
+
+  - match: { hits.total: 3 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-doc-values
+        body:
+          query:
+            range: {
+              boolean: {
+                lte: true,
+                gte: false
+              },
+            }
+
+  - match: { hits.total: 3 }
+
+  - do:
+      search:
+        rest_total_hits_as_int: true
+        index: test-doc-values
+        body:
+          query:
+            range: {
+              boolean: {
+                lte: false,
+                gte: true
+              },
+            }
+
+  - match: { hits.total: 0 }
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/360_from_and_size.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/360_from_and_size.yml
new file mode 100644
index 0000000000000..7f3fb77b86366
--- /dev/null
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/360_from_and_size.yml
@@ -0,0 +1,113 @@
+setup:
+  - do:
+      indices.create:
+          index:  test_1
+  - do:
+      index:
+          index:  test_1
+          id:     1
+          body:   { foo: bar }
+  - do:
+      index:
+          index:  test_1
+          id:     2
+          body:   { foo: bar }
+  - do:
+      index:
+          index:  test_1
+          id:     3
+          body:   { foo: bar }
+
+  - do:
+      index:
+          index:  test_1
+          id:     4
+          body:   { foo: bar }
+  - do:
+      indices.refresh:
+          index: [test_1]
+
+---
+teardown:
+  - do:
+      indices.delete:
+        index: test_1
+        ignore: 404
+
+---
+"Throws exception if from or size query parameter is negative":
+  - skip:
+      version: " - 2.13.99"
+      reason: "fixed in 2.14.0"
+  - do:
+      catch: '/\[from\] parameter cannot be negative, found \[-5\]/'
+      search:
+        index: test_1
+        from: -5
+        size: 10
+        body:
+          query:
+            match:
+              foo: bar
+
+  - do:
+      catch: '/\[size\] parameter cannot be negative, found \[-1\]/'
+      search:
+        index: test_1
+        from: 0
+        size: -1
+        body:
+          query:
+            match:
+              foo: bar
+
+  - do:
+      search:
+        index: test_1
+        from: 0
+        size: 10
+        body:
+          query:
+            match:
+              foo: bar
+
+  - match: {hits.total.value: 4}
+
+---
+"Throws exception if from or size request body parameter is negative":
+  - skip:
+      version: " - 2.13.99"
+      reason: "fixed in 2.14.0"
+  - do:
+      catch: '/\[from\] parameter cannot be negative, found \[-5\]/'
+      search:
+        index: test_1
+        body:
+          from: -5
+          size: 10
+          query:
+            match:
+              foo: bar
+
+  - do:
+      catch: '/\[size\] parameter cannot be negative, found \[-1\]/'
+      search:
+        index: test_1
+        body:
+          from: 0
+          size: -1
+          query:
+            match:
+              foo: bar
+
+  - do:
+      search:
+        index: test_1
+        body:
+          from: 0
+          size: 10
+          query:
+            match:
+              foo: bar
+
+  - match: {hits.total.value: 4}
diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search_pipeline/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search_pipeline/10_basic.yml
index 60c0706415bc2..a4877975a0052 100644
--- a/rest-api-spec/src/main/resources/rest-api-spec/test/search_pipeline/10_basic.yml
+++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search_pipeline/10_basic.yml
@@ -1,8 +1,8 @@
 ---
 "Test basic pipeline crud":
   - skip:
-      version: " - 2.6.99"
-      reason: "Added in 2.7.0"
+      version: " - 2.8.99"
+      reason: "Added in 2.9.0"
   - do:
       search_pipeline.put:
         id: "my_pipeline"
@@ -32,8 +32,8 @@
 ---
 "Test Put Versioned Pipeline":
   - skip:
-      version: " - 2.6.99"
-      reason: "Added in 2.7.0"
+      version: " - 2.8.99"
+      reason: "Added in 2.9.0"
   - do:
       search_pipeline.put:
         id: "my_pipeline"
@@ -125,8 +125,8 @@
 ---
 "Test Get All Pipelines":
   - skip:
-      version: " - 2.6.99"
-      reason: "Added in 2.7.0"
+      version: " - 2.8.99"
+      reason: "Added in 2.9.0"
   - do:
       search_pipeline.put:
         id: "first_pipeline"
@@ -152,8 +152,8 @@
 ---
 "Test invalid config":
   - skip:
-      version: " - 2.6.99"
-      reason: "Added in 2.7.0"
+      version: " - 2.8.99"
+      reason: "Added in 2.9.0"
   - do:
       catch: /parse_exception/
       search_pipeline.put:
diff --git a/server/build.gradle b/server/build.gradle
index e36498bf1038b..9714f13ec67d6 100644
--- a/server/build.gradle
+++ b/server/build.gradle
@@ -36,6 +36,7 @@ plugins {
   id('opensearch.publish')
   id('opensearch.internal-cluster-test')
   id('opensearch.optional-dependencies')
+  id('me.champeau.gradle.japicmp') version '0.4.2'
 }
 
 publishing {
@@ -97,7 +98,7 @@ dependencies {
   // percentiles aggregation
   api 'com.tdunning:t-digest:3.3'
   // precentil ranks aggregation
-  api 'org.hdrhistogram:HdrHistogram:2.1.12'
+  api 'org.hdrhistogram:HdrHistogram:2.2.1'
 
   // lucene spatial
   api "org.locationtech.spatial4j:spatial4j:${versions.spatial4j}", optional
@@ -141,7 +142,7 @@ tasks.withType(JavaCompile).configureEach {
 
 compileJava {
   options.compilerArgs += ['-processor', ['org.apache.logging.log4j.core.config.plugins.processor.PluginProcessor',
-    'org.opensearch.common.annotation.processor.ApiAnnotationProcessor'].join(','), '-AcontinueOnFailingChecks']
+    'org.opensearch.common.annotation.processor.ApiAnnotationProcessor'].join(',')]
 }
 
 tasks.named("internalClusterTest").configure {
@@ -172,6 +173,22 @@ tasks.named("testingConventions").configure {
     }
 }
 
+// Set to current version by default
+def japicmpCompareTarget = System.getProperty("japicmp.compare.version")
+if (japicmpCompareTarget == null) { /* use latest released version */
+    // Read the list from maven central.
+    // Fetch the metadata and parse the xml into Version instances, pick the latest one
+    japicmpCompareTarget = new URL('https://repo1.maven.org/maven2/org/opensearch/opensearch/maven-metadata.xml').openStream().withStream { s ->
+        new XmlParser().parse(s)
+           .versioning.versions.version
+           .collect { it.text() }.findAll { it ==~ /\d+\.\d+\.\d+/ }
+           .collect { org.opensearch.gradle.Version.fromString(it) }
+           .toSorted()
+           .last()
+           .toString()
+        }
+}
+
 def generateModulesList = tasks.register("generateModulesList") {
     List<String> modules = project(':modules').subprojects.collect { it.name }
     File modulesFile = new File(buildDir, 'generated-resources/modules.txt')
@@ -378,3 +395,87 @@ tasks.named("sourcesJar").configure {
     duplicatesStrategy = DuplicatesStrategy.EXCLUDE
   }
 }
+
+/** Compares the current build against a laltest released version or the version supplied through 'japicmp.compare.version' system property */
+tasks.register("japicmp", me.champeau.gradle.japicmp.JapicmpTask) {
+    logger.info("Comparing public APIs from ${version} to ${japicmpCompareTarget}")
+    oldClasspath.from(files("${buildDir}/japicmp-target/opensearch-${japicmpCompareTarget}.jar"))
+    newClasspath.from(tasks.named('jar'))
+    onlyModified = true
+    failOnModification = true
+    ignoreMissingClasses = true
+    annotationIncludes = ['@org.opensearch.common.annotation.PublicApi', '@org.opensearch.common.annotation.DeprecatedApi']
+    txtOutputFile = layout.buildDirectory.file("reports/java-compatibility/report.txt")
+    htmlOutputFile = layout.buildDirectory.file("reports/java-compatibility/report.html")
+    dependsOn downloadJapicmpCompareTarget
+}
+
+/** If the Java API Comparison task failed, print a hint if the change should be merged from its target branch */
+gradle.taskGraph.afterTask { Task task, TaskState state ->
+    if (task.name == 'japicmp' && state.failure != null) {
+        logger.info("Public APIs changes incompatiable with ${japicmpCompareTarget} target have been detected")
+    }
+}
+
+/** Downloads latest released version from maven repository */
+tasks.register("downloadJapicmpCompareTarget", Copy) {
+    def mavenSnapshotRepoUrl = "https://aws.oss.sonatype.org/content/repositories/snapshots/"
+    def groupId = "org.opensearch"
+    def artifactId = "opensearch"
+
+    // Add repository for snapshot artifacts if japicmp compare target version is snapshot
+    if (japicmpCompareTarget.endsWith("-SNAPSHOT")) {
+        def repos = project.getRepositories();
+        MavenArtifactRepository opensearchRepo = repos.maven(repo -> {
+            repo.setName("opensearch-snapshots");
+            repo.setUrl(mavenSnapshotRepoUrl);
+        });
+
+        repos.exclusiveContent(exclusiveRepo -> {
+            exclusiveRepo.filter(descriptor -> descriptor.includeGroup(groupId));
+            exclusiveRepo.forRepositories(opensearchRepo);
+        });
+    }
+
+    configurations {
+        japicmpCompareTargetArtifact {
+            exclude group: 'org.apache.lucene'
+        }
+    }
+
+    dependencies {
+        japicmpCompareTargetArtifact("${groupId}:${artifactId}:${japicmpCompareTarget}:")
+    }
+
+    from configurations.japicmpCompareTargetArtifact
+    into "$buildDir/japicmp-target"
+}
+
+/** Check if the sha is in the current history */
+def inHistory(String sha) {
+    try {
+      def commandCheckSha = "git merge-base --is-ancestor ${sha} HEAD"
+      commandCheckSha.execute()
+      return true
+    } catch (Exception) {
+      return false
+    }
+}
+
+/** Extracts the Git SHA used to build a jar from its manifest */
+def getGitShaFromJar(String jarPath) {
+    def sha = ''
+    try {
+        // Open the JAR file
+        def jarFile = new java.util.jar.JarFile(jarPath)
+        // Get the manifest from the JAR file
+        def manifest = jarFile.manifest
+        def attributes = manifest.mainAttributes
+        // Assuming the Git SHA is stored under an attribute named 'Git-SHA'
+        sha = attributes.getValue('Change')
+        jarFile.close()
+    } catch (IOException e) {
+        println "Failed to read the JAR file: $e.message"
+    }
+    return sha
+}
diff --git a/server/licenses/HdrHistogram-2.1.12.jar.sha1 b/server/licenses/HdrHistogram-2.1.12.jar.sha1
deleted file mode 100644
index 9d20fa0e5f22d..0000000000000
--- a/server/licenses/HdrHistogram-2.1.12.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-6eb7552156e0d517ae80cc2247be1427c8d90452
\ No newline at end of file
diff --git a/server/licenses/HdrHistogram-2.2.1.jar.sha1 b/server/licenses/HdrHistogram-2.2.1.jar.sha1
new file mode 100644
index 0000000000000..68225950d4744
--- /dev/null
+++ b/server/licenses/HdrHistogram-2.2.1.jar.sha1
@@ -0,0 +1 @@
+0eb1feb351f64176c377772a30174e582c0274d5
\ No newline at end of file
diff --git a/server/licenses/joda-time-2.12.2.jar.sha1 b/server/licenses/joda-time-2.12.2.jar.sha1
deleted file mode 100644
index 6e9b28eb35597..0000000000000
--- a/server/licenses/joda-time-2.12.2.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-78e18a7b4180e911dafba0a412adfa82c1e3d14b
\ No newline at end of file
diff --git a/server/licenses/joda-time-2.12.7.jar.sha1 b/server/licenses/joda-time-2.12.7.jar.sha1
new file mode 100644
index 0000000000000..7ce5c501873c0
--- /dev/null
+++ b/server/licenses/joda-time-2.12.7.jar.sha1
@@ -0,0 +1 @@
+d015b997eccd511e5567218a51651ff0625f6f25
\ No newline at end of file
diff --git a/server/licenses/lucene-analysis-common-9.11.0-snapshot-8a555eb.jar.sha1 b/server/licenses/lucene-analysis-common-9.11.0-snapshot-8a555eb.jar.sha1
deleted file mode 100644
index 9cab77f4e7394..0000000000000
--- a/server/licenses/lucene-analysis-common-9.11.0-snapshot-8a555eb.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-6f0cfa07a5e4b36423e398cd1fd51c6825773d9c
\ No newline at end of file
diff --git a/server/licenses/lucene-analysis-common-9.11.0-snapshot-fb97840.jar.sha1 b/server/licenses/lucene-analysis-common-9.11.0-snapshot-fb97840.jar.sha1
new file mode 100644
index 0000000000000..f229c373aa1af
--- /dev/null
+++ b/server/licenses/lucene-analysis-common-9.11.0-snapshot-fb97840.jar.sha1
@@ -0,0 +1 @@
+cd0afb5da5dcb4c7498bd1ee7f7bab0e289404b8
\ No newline at end of file
diff --git a/server/licenses/lucene-backward-codecs-9.11.0-snapshot-8a555eb.jar.sha1 b/server/licenses/lucene-backward-codecs-9.11.0-snapshot-8a555eb.jar.sha1
deleted file mode 100644
index 2d5d1a281a0f0..0000000000000
--- a/server/licenses/lucene-backward-codecs-9.11.0-snapshot-8a555eb.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-ab201b997c8449db1ecd2fa88bd42d2f457286fa
\ No newline at end of file
diff --git a/server/licenses/lucene-backward-codecs-9.11.0-snapshot-fb97840.jar.sha1 b/server/licenses/lucene-backward-codecs-9.11.0-snapshot-fb97840.jar.sha1
new file mode 100644
index 0000000000000..edaf28a7f6e76
--- /dev/null
+++ b/server/licenses/lucene-backward-codecs-9.11.0-snapshot-fb97840.jar.sha1
@@ -0,0 +1 @@
+88888315cd60e565960ae2e6fed2af0df077a2a2
\ No newline at end of file
diff --git a/server/licenses/lucene-core-9.11.0-snapshot-8a555eb.jar.sha1 b/server/licenses/lucene-core-9.11.0-snapshot-8a555eb.jar.sha1
deleted file mode 100644
index 25beb34873c0c..0000000000000
--- a/server/licenses/lucene-core-9.11.0-snapshot-8a555eb.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-fe1cf5663be8bdb6aa757fd4101bc551684c90fb
\ No newline at end of file
diff --git a/server/licenses/lucene-core-9.11.0-snapshot-fb97840.jar.sha1 b/server/licenses/lucene-core-9.11.0-snapshot-fb97840.jar.sha1
new file mode 100644
index 0000000000000..88309bc46411a
--- /dev/null
+++ b/server/licenses/lucene-core-9.11.0-snapshot-fb97840.jar.sha1
@@ -0,0 +1 @@
+53a828e3e88f55c83979cd3df0704617cc9edb9a
\ No newline at end of file
diff --git a/server/licenses/lucene-grouping-9.11.0-snapshot-8a555eb.jar.sha1 b/server/licenses/lucene-grouping-9.11.0-snapshot-8a555eb.jar.sha1
deleted file mode 100644
index 27d4f0f5874e9..0000000000000
--- a/server/licenses/lucene-grouping-9.11.0-snapshot-8a555eb.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-a988f92842e48195c75a49377432533c9170d93d
\ No newline at end of file
diff --git a/server/licenses/lucene-grouping-9.11.0-snapshot-fb97840.jar.sha1 b/server/licenses/lucene-grouping-9.11.0-snapshot-fb97840.jar.sha1
new file mode 100644
index 0000000000000..13f1276e3b033
--- /dev/null
+++ b/server/licenses/lucene-grouping-9.11.0-snapshot-fb97840.jar.sha1
@@ -0,0 +1 @@
+d1f54a816c9d85e890a862a2dffdc734ece2770c
\ No newline at end of file
diff --git a/server/licenses/lucene-highlighter-9.11.0-snapshot-8a555eb.jar.sha1 b/server/licenses/lucene-highlighter-9.11.0-snapshot-8a555eb.jar.sha1
deleted file mode 100644
index 2545822f2ac7b..0000000000000
--- a/server/licenses/lucene-highlighter-9.11.0-snapshot-8a555eb.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-7c669e2c01565d3bdf175cd61a1e4d0bdfc44311
\ No newline at end of file
diff --git a/server/licenses/lucene-highlighter-9.11.0-snapshot-fb97840.jar.sha1 b/server/licenses/lucene-highlighter-9.11.0-snapshot-fb97840.jar.sha1
new file mode 100644
index 0000000000000..6cef51ac4453f
--- /dev/null
+++ b/server/licenses/lucene-highlighter-9.11.0-snapshot-fb97840.jar.sha1
@@ -0,0 +1 @@
+1e7c9336fa86fb866fcd76ea5d6283c804b4d580
\ No newline at end of file
diff --git a/server/licenses/lucene-join-9.11.0-snapshot-8a555eb.jar.sha1 b/server/licenses/lucene-join-9.11.0-snapshot-8a555eb.jar.sha1
deleted file mode 100644
index fe44ad772335f..0000000000000
--- a/server/licenses/lucene-join-9.11.0-snapshot-8a555eb.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-633a6d44b4cde8e149daa3407e8b8f644eece951
\ No newline at end of file
diff --git a/server/licenses/lucene-join-9.11.0-snapshot-fb97840.jar.sha1 b/server/licenses/lucene-join-9.11.0-snapshot-fb97840.jar.sha1
new file mode 100644
index 0000000000000..2524ac35c2afe
--- /dev/null
+++ b/server/licenses/lucene-join-9.11.0-snapshot-fb97840.jar.sha1
@@ -0,0 +1 @@
+17be4fc1f9feca0dac84a37d54dca4b32df4c619
\ No newline at end of file
diff --git a/server/licenses/lucene-memory-9.11.0-snapshot-8a555eb.jar.sha1 b/server/licenses/lucene-memory-9.11.0-snapshot-8a555eb.jar.sha1
deleted file mode 100644
index 85bfbe066ff56..0000000000000
--- a/server/licenses/lucene-memory-9.11.0-snapshot-8a555eb.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-25390259c9e5592354efbc2f250bb396402016b2
\ No newline at end of file
diff --git a/server/licenses/lucene-memory-9.11.0-snapshot-fb97840.jar.sha1 b/server/licenses/lucene-memory-9.11.0-snapshot-fb97840.jar.sha1
new file mode 100644
index 0000000000000..f5ef377300839
--- /dev/null
+++ b/server/licenses/lucene-memory-9.11.0-snapshot-fb97840.jar.sha1
@@ -0,0 +1 @@
+7350675a2cf386c0f003b667b61db614f03bb984
\ No newline at end of file
diff --git a/server/licenses/lucene-misc-9.11.0-snapshot-8a555eb.jar.sha1 b/server/licenses/lucene-misc-9.11.0-snapshot-8a555eb.jar.sha1
deleted file mode 100644
index 4dba5837b66de..0000000000000
--- a/server/licenses/lucene-misc-9.11.0-snapshot-8a555eb.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-056f87a3d30c223b08d2f45fe465ddf11210b85f
\ No newline at end of file
diff --git a/server/licenses/lucene-misc-9.11.0-snapshot-fb97840.jar.sha1 b/server/licenses/lucene-misc-9.11.0-snapshot-fb97840.jar.sha1
new file mode 100644
index 0000000000000..e94fcf0f259a1
--- /dev/null
+++ b/server/licenses/lucene-misc-9.11.0-snapshot-fb97840.jar.sha1
@@ -0,0 +1 @@
+c5c8bd120d5985ab6bd4e5f89efe08c226c0a323
\ No newline at end of file
diff --git a/server/licenses/lucene-queries-9.11.0-snapshot-8a555eb.jar.sha1 b/server/licenses/lucene-queries-9.11.0-snapshot-8a555eb.jar.sha1
deleted file mode 100644
index b6b8c441eefb1..0000000000000
--- a/server/licenses/lucene-queries-9.11.0-snapshot-8a555eb.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-e640f850b4fb13190be8422fe74c14c9d6603bb5
\ No newline at end of file
diff --git a/server/licenses/lucene-queries-9.11.0-snapshot-fb97840.jar.sha1 b/server/licenses/lucene-queries-9.11.0-snapshot-fb97840.jar.sha1
new file mode 100644
index 0000000000000..fc80394195fa9
--- /dev/null
+++ b/server/licenses/lucene-queries-9.11.0-snapshot-fb97840.jar.sha1
@@ -0,0 +1 @@
+2e1975ac26e9172722f734bf0f5583317e5eb16a
\ No newline at end of file
diff --git a/server/licenses/lucene-queryparser-9.11.0-snapshot-8a555eb.jar.sha1 b/server/licenses/lucene-queryparser-9.11.0-snapshot-8a555eb.jar.sha1
deleted file mode 100644
index d0e77b04db51a..0000000000000
--- a/server/licenses/lucene-queryparser-9.11.0-snapshot-8a555eb.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-8eb57762bf408fa51d7511f5e3b917627be61d1d
\ No newline at end of file
diff --git a/server/licenses/lucene-queryparser-9.11.0-snapshot-fb97840.jar.sha1 b/server/licenses/lucene-queryparser-9.11.0-snapshot-fb97840.jar.sha1
new file mode 100644
index 0000000000000..10ef577bc1bdc
--- /dev/null
+++ b/server/licenses/lucene-queryparser-9.11.0-snapshot-fb97840.jar.sha1
@@ -0,0 +1 @@
+c8cf3d5dd4d0538b38e4e88bb865bc59d835d887
\ No newline at end of file
diff --git a/server/licenses/lucene-sandbox-9.11.0-snapshot-8a555eb.jar.sha1 b/server/licenses/lucene-sandbox-9.11.0-snapshot-8a555eb.jar.sha1
deleted file mode 100644
index 179df9f07a594..0000000000000
--- a/server/licenses/lucene-sandbox-9.11.0-snapshot-8a555eb.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-615b4a90c5402084c2d5916a4c1fadc9d9177782
\ No newline at end of file
diff --git a/server/licenses/lucene-sandbox-9.11.0-snapshot-fb97840.jar.sha1 b/server/licenses/lucene-sandbox-9.11.0-snapshot-fb97840.jar.sha1
new file mode 100644
index 0000000000000..08a61ba30bc0d
--- /dev/null
+++ b/server/licenses/lucene-sandbox-9.11.0-snapshot-fb97840.jar.sha1
@@ -0,0 +1 @@
+5a3a7a138ff4978f3ddb186d9786e6cb4793b291
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial-extras-9.11.0-snapshot-8a555eb.jar.sha1 b/server/licenses/lucene-spatial-extras-9.11.0-snapshot-8a555eb.jar.sha1
deleted file mode 100644
index 9b88b24c21b12..0000000000000
--- a/server/licenses/lucene-spatial-extras-9.11.0-snapshot-8a555eb.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-19b42cdb5f76f63dece3ef5128207ebdd3741d48
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial-extras-9.11.0-snapshot-fb97840.jar.sha1 b/server/licenses/lucene-spatial-extras-9.11.0-snapshot-fb97840.jar.sha1
new file mode 100644
index 0000000000000..a244219c1de60
--- /dev/null
+++ b/server/licenses/lucene-spatial-extras-9.11.0-snapshot-fb97840.jar.sha1
@@ -0,0 +1 @@
+dc71e0125c66d29a1bffc1ddeab4b96526e737c8
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial3d-9.11.0-snapshot-8a555eb.jar.sha1 b/server/licenses/lucene-spatial3d-9.11.0-snapshot-8a555eb.jar.sha1
deleted file mode 100644
index 789ab1d52ea8c..0000000000000
--- a/server/licenses/lucene-spatial3d-9.11.0-snapshot-8a555eb.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-882691fe917e716fe6bcf8c0dd984b153495d015
\ No newline at end of file
diff --git a/server/licenses/lucene-spatial3d-9.11.0-snapshot-fb97840.jar.sha1 b/server/licenses/lucene-spatial3d-9.11.0-snapshot-fb97840.jar.sha1
new file mode 100644
index 0000000000000..d2b3821bbf5f6
--- /dev/null
+++ b/server/licenses/lucene-spatial3d-9.11.0-snapshot-fb97840.jar.sha1
@@ -0,0 +1 @@
+aef4c04d353092a438eee302521fe34188b7c4df
\ No newline at end of file
diff --git a/server/licenses/lucene-suggest-9.11.0-snapshot-8a555eb.jar.sha1 b/server/licenses/lucene-suggest-9.11.0-snapshot-8a555eb.jar.sha1
deleted file mode 100644
index 8cfaf60763724..0000000000000
--- a/server/licenses/lucene-suggest-9.11.0-snapshot-8a555eb.jar.sha1
+++ /dev/null
@@ -1 +0,0 @@
-ef6d483960f776d5dbdd1009863786ee09ba5707
\ No newline at end of file
diff --git a/server/licenses/lucene-suggest-9.11.0-snapshot-fb97840.jar.sha1 b/server/licenses/lucene-suggest-9.11.0-snapshot-fb97840.jar.sha1
new file mode 100644
index 0000000000000..2c147e4651a44
--- /dev/null
+++ b/server/licenses/lucene-suggest-9.11.0-snapshot-fb97840.jar.sha1
@@ -0,0 +1 @@
+86f68cacd85f99b4ddcda3aff7c873349ba59381
\ No newline at end of file
diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/RemoteCloneIndexIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/RemoteCloneIndexIT.java
index f50e8fd0a38cf..4be049c9a9109 100644
--- a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/RemoteCloneIndexIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/create/RemoteCloneIndexIT.java
@@ -40,13 +40,17 @@
  */
 
 import org.opensearch.Version;
+import org.opensearch.action.admin.cluster.health.ClusterHealthRequest;
+import org.opensearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse;
 import org.opensearch.action.admin.indices.shrink.ResizeType;
 import org.opensearch.action.admin.indices.stats.IndicesStatsResponse;
+import org.opensearch.client.Requests;
 import org.opensearch.cluster.routing.allocation.decider.EnableAllocationDecider;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.xcontent.MediaTypeRegistry;
 import org.opensearch.index.query.TermsQueryBuilder;
+import org.opensearch.indices.recovery.RecoverySettings;
 import org.opensearch.remotestore.RemoteStoreBaseIntegTestCase;
 import org.opensearch.test.VersionUtils;
 
@@ -125,7 +129,9 @@ public void testCreateCloneIndex() {
                 .cluster()
                 .prepareUpdateSettings()
                 .setTransientSettings(
-                    Settings.builder().put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), (String) null)
+                    Settings.builder()
+                        .put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), (String) null)
+                        .put(RecoverySettings.INDICES_INTERNAL_REMOTE_UPLOAD_TIMEOUT.getKey(), (String) null)
                 )
                 .get();
         }
@@ -156,7 +162,11 @@ public void testCreateCloneIndexFailure() throws ExecutionException, Interrupted
         client().admin()
             .cluster()
             .prepareUpdateSettings()
-            .setTransientSettings(Settings.builder().put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), "none"))
+            .setTransientSettings(
+                Settings.builder()
+                    .put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), "none")
+                    .put(RecoverySettings.INDICES_INTERNAL_REMOTE_UPLOAD_TIMEOUT.getKey(), "10s")
+            )
             .get();
         try {
             setFailRate(REPOSITORY_NAME, 100);
@@ -168,9 +178,14 @@ public void testCreateCloneIndexFailure() throws ExecutionException, Interrupted
                 .setWaitForActiveShards(0)
                 .setSettings(Settings.builder().put("index.number_of_replicas", 0).putNull("index.blocks.write").build())
                 .get();
-
-            Thread.sleep(2000);
-            ensureYellow("target");
+            // waiting more than waitForRemoteStoreSync's sleep time of 30 sec to deterministically fail
+            Thread.sleep(40000);
+            ensureRed("target");
+            ClusterHealthRequest healthRequest = Requests.clusterHealthRequest()
+                .waitForNoRelocatingShards(true)
+                .waitForNoInitializingShards(true);
+            ClusterHealthResponse actionGet = client().admin().cluster().health(healthRequest).actionGet();
+            assertEquals(actionGet.getUnassignedShards(), numPrimaryShards);
 
         } catch (ExecutionException | InterruptedException e) {
             throw new RuntimeException(e);
@@ -182,11 +197,12 @@ public void testCreateCloneIndexFailure() throws ExecutionException, Interrupted
                 .cluster()
                 .prepareUpdateSettings()
                 .setTransientSettings(
-                    Settings.builder().put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), (String) null)
+                    Settings.builder()
+                        .put(EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), (String) null)
+                        .put(RecoverySettings.INDICES_INTERNAL_REMOTE_UPLOAD_TIMEOUT.getKey(), (String) null)
                 )
                 .get();
         }
-
     }
 
 }
diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/state/FetchByTermVersionIT.java b/server/src/internalClusterTest/java/org/opensearch/action/support/clustermanager/term/FetchByTermVersionIT.java
similarity index 97%
rename from server/src/internalClusterTest/java/org/opensearch/cluster/state/FetchByTermVersionIT.java
rename to server/src/internalClusterTest/java/org/opensearch/action/support/clustermanager/term/FetchByTermVersionIT.java
index cef184b3fddf9..72dcc98dcdc12 100644
--- a/server/src/internalClusterTest/java/org/opensearch/cluster/state/FetchByTermVersionIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/action/support/clustermanager/term/FetchByTermVersionIT.java
@@ -6,12 +6,10 @@
  * compatible open source license.
  */
 
-package org.opensearch.cluster.state;
+package org.opensearch.action.support.clustermanager.term;
 
 import org.opensearch.action.admin.cluster.state.ClusterStateRequest;
 import org.opensearch.action.admin.cluster.state.ClusterStateResponse;
-import org.opensearch.action.admin.cluster.state.term.GetTermVersionAction;
-import org.opensearch.action.admin.cluster.state.term.GetTermVersionResponse;
 import org.opensearch.action.admin.indices.mapping.put.PutMappingRequest;
 import org.opensearch.cluster.ClusterName;
 import org.opensearch.cluster.coordination.ClusterStateTermVersion;
diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/ClusterHealthIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/ClusterHealthIT.java
index d63b87cbee6f7..0304e00a49070 100644
--- a/server/src/internalClusterTest/java/org/opensearch/cluster/ClusterHealthIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/cluster/ClusterHealthIT.java
@@ -377,7 +377,7 @@ public void testHealthOnClusterManagerFailover() throws Exception {
                     .prepareHealth()
                     .setWaitForEvents(Priority.LANGUID)
                     .setWaitForGreenStatus()
-                    .setClusterManagerNodeTimeout(TimeValue.timeValueMinutes(2))
+                    .setClusterManagerNodeTimeout(TimeValue.timeValueMinutes(3))
                     .execute()
             );
             internalCluster().restartNode(internalCluster().getClusterManagerName(), InternalTestCluster.EMPTY_CALLBACK);
diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/metadata/ClusterIndexRefreshIntervalIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/metadata/ClusterIndexRefreshIntervalIT.java
index 25fa7ae7eb8eb..f936b53f52a7b 100644
--- a/server/src/internalClusterTest/java/org/opensearch/cluster/metadata/ClusterIndexRefreshIntervalIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/cluster/metadata/ClusterIndexRefreshIntervalIT.java
@@ -235,33 +235,19 @@ public void testDefaultRefreshIntervalWithUpdateClusterAndIndexSettings() throws
     }
 
     public void testRefreshIntervalDisabled() throws ExecutionException, InterruptedException {
-        TimeValue clusterMinimumRefreshInterval = client().settings()
-            .getAsTime(IndicesService.CLUSTER_MINIMUM_INDEX_REFRESH_INTERVAL_SETTING.getKey(), TimeValue.MINUS_ONE);
-        boolean createIndexSuccess = clusterMinimumRefreshInterval.equals(TimeValue.MINUS_ONE);
         String clusterManagerName = internalCluster().getClusterManagerName();
         List<String> dataNodes = new ArrayList<>(internalCluster().getDataNodeNames());
         Settings settings = Settings.builder()
             .put(indexSettings())
             .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), IndexSettings.MINIMUM_REFRESH_INTERVAL)
             .build();
-        if (createIndexSuccess) {
-            createIndex(INDEX_NAME, settings);
-            ensureYellowAndNoInitializingShards(INDEX_NAME);
-            ensureGreen(INDEX_NAME);
-            GetIndexResponse getIndexResponse = client(clusterManagerName).admin().indices().getIndex(new GetIndexRequest()).get();
-            IndicesService indicesService = internalCluster().getInstance(IndicesService.class, randomFrom(dataNodes));
-            String uuid = getIndexResponse.getSettings().get(INDEX_NAME).get(IndexMetadata.SETTING_INDEX_UUID);
-            IndexService indexService = indicesService.indexService(new Index(INDEX_NAME, uuid));
-            assertEquals(IndexSettings.MINIMUM_REFRESH_INTERVAL, indexService.getRefreshTaskInterval());
-        } else {
-            IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, () -> createIndex(INDEX_NAME, settings));
-            assertEquals(
-                "invalid index.refresh_interval [-1]: cannot be smaller than cluster.minimum.index.refresh_interval ["
-                    + getMinRefreshIntervalForRefreshDisabled()
-                    + "]",
-                exception.getMessage()
-            );
-        }
+        createIndex(INDEX_NAME, settings);
+        ensureGreen(INDEX_NAME);
+        GetIndexResponse getIndexResponse = client(clusterManagerName).admin().indices().getIndex(new GetIndexRequest()).get();
+        IndicesService indicesService = internalCluster().getInstance(IndicesService.class, randomFrom(dataNodes));
+        String uuid = getIndexResponse.getSettings().get(INDEX_NAME).get(IndexMetadata.SETTING_INDEX_UUID);
+        IndexService indexService = indicesService.indexService(new Index(INDEX_NAME, uuid));
+        assertEquals(IndexSettings.MINIMUM_REFRESH_INTERVAL, indexService.getRefreshTaskInterval());
     }
 
     protected TimeValue getMinRefreshIntervalForRefreshDisabled() {
@@ -366,6 +352,147 @@ public void testClusterMinimumChangeOnIndexWithCustomRefreshInterval() throws Ex
         assertEquals(customRefreshInterval, indexService.getRefreshTaskInterval());
     }
 
+    public void testClusterMinimumRefreshIntervalOfMinusOneFails() {
+        // This test checks that we can not set cluster minimum refresh interval as -1 (or -1ms).
+        String clusterManagerName = internalCluster().getClusterManagerName();
+        String refreshInterval = randomFrom("-1", "-1ms");
+        IllegalArgumentException ex = assertThrows(
+            IllegalArgumentException.class,
+            () -> client(clusterManagerName).admin()
+                .cluster()
+                .prepareUpdateSettings()
+                .setTransientSettings(Settings.builder().put(CLUSTER_MINIMUM_INDEX_REFRESH_INTERVAL_SETTING.getKey(), refreshInterval))
+                .get()
+        );
+        assertEquals(
+            "failed to parse value [" + refreshInterval + "] for setting [cluster.minimum.index.refresh_interval], must be >= [0ms]",
+            ex.getMessage()
+        );
+    }
+
+    public void testClusterMinimumRefreshIntervalOfZero() {
+        // This test checks that we can set the cluster minimum refresh interval as 0.
+        String clusterManagerName = internalCluster().getClusterManagerName();
+        client(clusterManagerName).admin()
+            .cluster()
+            .prepareUpdateSettings()
+            .setTransientSettings(Settings.builder().put(CLUSTER_MINIMUM_INDEX_REFRESH_INTERVAL_SETTING.getKey(), "0"))
+            .get();
+    }
+
+    public void testDefaultRefreshIntervalOfMinusOneIrrespectiveOfMinimum() {
+        // This test checks that we are able to set the cluster default refresh interval to one regardless of what the
+        // minimum is set to. -1 corresponds to no period background refreshes.
+        String clusterManagerName = internalCluster().getClusterManagerName();
+        client(clusterManagerName).admin()
+            .cluster()
+            .prepareUpdateSettings()
+            .setTransientSettings(
+                Settings.builder()
+                    .put(CLUSTER_MINIMUM_INDEX_REFRESH_INTERVAL_SETTING.getKey(), randomFrom("0", "1ms", "1s", "10s"))
+                    .put(CLUSTER_DEFAULT_INDEX_REFRESH_INTERVAL_SETTING.getKey(), randomFrom("-1", "-1ms"))
+            )
+            .get();
+    }
+
+    public void testCreateIndexWithMinusOneRefreshInterval() throws ExecutionException, InterruptedException {
+        // This test checks that we are able to create index with -1 refresh interval using index settings and default interval both.
+        String clusterManagerName = internalCluster().getClusterManagerName();
+        client(clusterManagerName).admin()
+            .cluster()
+            .prepareUpdateSettings()
+            .setTransientSettings(
+                Settings.builder()
+                    .put(CLUSTER_MINIMUM_INDEX_REFRESH_INTERVAL_SETTING.getKey(), "10s")
+                    .put(CLUSTER_DEFAULT_INDEX_REFRESH_INTERVAL_SETTING.getKey(), "10s")
+            )
+            .get();
+
+        Settings indexSettings = Settings.builder()
+            .put(indexSettings())
+            .put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), randomFrom("-1", "-1ms"))
+            .build();
+        createIndex(INDEX_NAME, indexSettings);
+        ensureGreen(INDEX_NAME);
+
+        IndexService indexService = getIndexServiceFromRandomDataNode(INDEX_NAME);
+        assertEquals(-1, indexService.getRefreshTaskInterval().millis());
+
+        client(clusterManagerName).admin()
+            .cluster()
+            .prepareUpdateSettings()
+            .setTransientSettings(Settings.builder().put(CLUSTER_DEFAULT_INDEX_REFRESH_INTERVAL_SETTING.getKey(), randomFrom("-1", "-1ms")))
+            .get();
+        createIndex(OTHER_INDEX_NAME);
+        ensureGreen(OTHER_INDEX_NAME);
+        indexService = getIndexServiceFromRandomDataNode(OTHER_INDEX_NAME);
+        assertEquals(-1, indexService.getRefreshTaskInterval().millis());
+    }
+
+    public void testUpdateIndexWithMinusOneRefreshInterval() throws ExecutionException, InterruptedException {
+        // This test checks that we are able to update index with -1 refresh interval using index settings and default interval both.
+        String clusterManagerName = internalCluster().getClusterManagerName();
+        client(clusterManagerName).admin()
+            .cluster()
+            .prepareUpdateSettings()
+            .setTransientSettings(
+                Settings.builder()
+                    .put(CLUSTER_MINIMUM_INDEX_REFRESH_INTERVAL_SETTING.getKey(), "10s")
+                    .put(CLUSTER_DEFAULT_INDEX_REFRESH_INTERVAL_SETTING.getKey(), "10s")
+            )
+            .get();
+
+        createIndex(INDEX_NAME);
+        ensureGreen(INDEX_NAME);
+        IndexService indexService = getIndexServiceFromRandomDataNode(INDEX_NAME);
+        assertEquals(10, indexService.getRefreshTaskInterval().seconds());
+
+        client(clusterManagerName).admin()
+            .indices()
+            .updateSettings(
+                new UpdateSettingsRequest(INDEX_NAME).settings(
+                    Settings.builder().put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), randomFrom("-1", "-1ms"))
+                )
+            )
+            .actionGet();
+        assertEquals(-1, indexService.getRefreshTaskInterval().millis());
+
+        client(clusterManagerName).admin()
+            .indices()
+            .updateSettings(
+                new UpdateSettingsRequest(INDEX_NAME).settings(
+                    Settings.builder().put(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey(), "100s")
+                )
+            )
+            .actionGet();
+        assertEquals(100, indexService.getRefreshTaskInterval().seconds());
+
+        client(clusterManagerName).admin()
+            .cluster()
+            .prepareUpdateSettings()
+            .setTransientSettings(Settings.builder().put(CLUSTER_DEFAULT_INDEX_REFRESH_INTERVAL_SETTING.getKey(), randomFrom("-1", "-1ms")))
+            .get();
+
+        client(clusterManagerName).admin()
+            .indices()
+            .updateSettings(
+                new UpdateSettingsRequest(INDEX_NAME).settings(
+                    Settings.builder().putNull(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey())
+                )
+            )
+            .actionGet();
+        assertEquals(-1, indexService.getRefreshTaskInterval().millis());
+    }
+
+    private IndexService getIndexServiceFromRandomDataNode(String indexName) throws ExecutionException, InterruptedException {
+        String clusterManagerName = internalCluster().getClusterManagerName();
+        List<String> dataNodes = new ArrayList<>(internalCluster().getDataNodeNames());
+        GetIndexResponse getIndexResponse = client(clusterManagerName).admin().indices().getIndex(new GetIndexRequest()).get();
+        IndicesService indicesService = internalCluster().getInstance(IndicesService.class, randomFrom(dataNodes));
+        String uuid = getIndexResponse.getSettings().get(indexName).get(IndexMetadata.SETTING_INDEX_UUID);
+        return indicesService.indexService(new Index(indexName, uuid));
+    }
+
     protected TimeValue getDefaultRefreshInterval() {
         return IndexSettings.DEFAULT_REFRESH_INTERVAL;
     }
diff --git a/server/src/internalClusterTest/java/org/opensearch/discovery/ClusterManagerDisruptionIT.java b/server/src/internalClusterTest/java/org/opensearch/discovery/ClusterManagerDisruptionIT.java
index 79f6ba6dfa642..65ca8eab278ef 100644
--- a/server/src/internalClusterTest/java/org/opensearch/discovery/ClusterManagerDisruptionIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/discovery/ClusterManagerDisruptionIT.java
@@ -198,11 +198,15 @@ public void testIsolateClusterManagerAndVerifyClusterStateConsensus() throws Exc
                             + nodeState
                     );
                 }
-
             }
-            ClusterStateStats clusterStateStats = internalCluster().clusterService().getClusterManagerService().getClusterStateStats();
-            assertTrue(clusterStateStats.getUpdateFailed() > 0);
+
         });
+
+        ClusterStateStats clusterStateStats = internalCluster().clusterService(isolatedNode)
+            .getClusterManagerService()
+            .getClusterStateStats();
+        assertTrue(clusterStateStats.getUpdateFailed() > 0);
+
     }
 
     /**
diff --git a/server/src/internalClusterTest/java/org/opensearch/discovery/StableClusterManagerDisruptionIT.java b/server/src/internalClusterTest/java/org/opensearch/discovery/StableClusterManagerDisruptionIT.java
index f6f2b2dbfd096..4de79a31d0146 100644
--- a/server/src/internalClusterTest/java/org/opensearch/discovery/StableClusterManagerDisruptionIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/discovery/StableClusterManagerDisruptionIT.java
@@ -203,7 +203,7 @@ public void testStaleClusterManagerNotHijackingMajority() throws Exception {
             3,
             Settings.builder()
                 .put(LeaderChecker.LEADER_CHECK_TIMEOUT_SETTING.getKey(), "1s")
-                .put(Coordinator.PUBLISH_TIMEOUT_SETTING.getKey(), "1s")
+                .put(Coordinator.PUBLISH_TIMEOUT_SETTING.getKey(), "2s")
                 .build()
         );
         ensureStableCluster(3);
diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/GatewayRecoveryTestUtils.java b/server/src/internalClusterTest/java/org/opensearch/gateway/GatewayRecoveryTestUtils.java
index 2b6a5b4ee6867..dc157681be6fa 100644
--- a/server/src/internalClusterTest/java/org/opensearch/gateway/GatewayRecoveryTestUtils.java
+++ b/server/src/internalClusterTest/java/org/opensearch/gateway/GatewayRecoveryTestUtils.java
@@ -54,7 +54,7 @@ public static Map<ShardId, ShardAttributes> prepareRequestMap(String[] indices,
             );
             for (int shardIdNum = 0; shardIdNum < primaryShardCount; shardIdNum++) {
                 final ShardId shardId = new ShardId(index, shardIdNum);
-                shardIdShardAttributesMap.put(shardId, new ShardAttributes(shardId, customDataPath));
+                shardIdShardAttributesMap.put(shardId, new ShardAttributes(customDataPath));
             }
         }
         return shardIdShardAttributesMap;
diff --git a/server/src/internalClusterTest/java/org/opensearch/gateway/RecoveryFromGatewayIT.java b/server/src/internalClusterTest/java/org/opensearch/gateway/RecoveryFromGatewayIT.java
index ba03532a9aa2f..bc0557ddc2afa 100644
--- a/server/src/internalClusterTest/java/org/opensearch/gateway/RecoveryFromGatewayIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/gateway/RecoveryFromGatewayIT.java
@@ -32,24 +32,31 @@
 
 package org.opensearch.gateway;
 
+import org.apache.lucene.index.CorruptIndexException;
 import org.opensearch.Version;
 import org.opensearch.action.admin.cluster.configuration.AddVotingConfigExclusionsAction;
 import org.opensearch.action.admin.cluster.configuration.AddVotingConfigExclusionsRequest;
 import org.opensearch.action.admin.cluster.configuration.ClearVotingConfigExclusionsAction;
 import org.opensearch.action.admin.cluster.configuration.ClearVotingConfigExclusionsRequest;
+import org.opensearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.opensearch.action.admin.cluster.reroute.ClusterRerouteResponse;
 import org.opensearch.action.admin.cluster.shards.ClusterSearchShardsGroup;
 import org.opensearch.action.admin.cluster.shards.ClusterSearchShardsResponse;
+import org.opensearch.action.admin.indices.exists.indices.IndicesExistsResponse;
 import org.opensearch.action.admin.indices.recovery.RecoveryResponse;
 import org.opensearch.action.admin.indices.stats.IndexStats;
+import org.opensearch.action.admin.indices.stats.IndicesStatsResponse;
 import org.opensearch.action.admin.indices.stats.ShardStats;
 import org.opensearch.action.support.ActionTestUtils;
+import org.opensearch.action.support.master.AcknowledgedResponse;
+import org.opensearch.client.Requests;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.coordination.ElectionSchedulerFactory;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.cluster.routing.ShardRouting;
 import org.opensearch.cluster.routing.UnassignedInfo;
+import org.opensearch.cluster.routing.allocation.ExistingShardsAllocator;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.xcontent.XContentFactory;
@@ -62,6 +69,7 @@
 import org.opensearch.index.MergePolicyProvider;
 import org.opensearch.index.engine.Engine;
 import org.opensearch.index.query.QueryBuilders;
+import org.opensearch.index.shard.IndexShard;
 import org.opensearch.index.shard.ShardPath;
 import org.opensearch.indices.IndicesService;
 import org.opensearch.indices.recovery.RecoveryState;
@@ -71,6 +79,7 @@
 import org.opensearch.indices.store.TransportNodesListShardStoreMetadataHelper;
 import org.opensearch.plugins.Plugin;
 import org.opensearch.test.InternalSettingsPlugin;
+import org.opensearch.test.InternalTestCluster;
 import org.opensearch.test.InternalTestCluster.RestartCallback;
 import org.opensearch.test.OpenSearchIntegTestCase;
 import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope;
@@ -94,6 +103,8 @@
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.emptySet;
 import static org.opensearch.cluster.coordination.ClusterBootstrapService.INITIAL_CLUSTER_MANAGER_NODES_SETTING;
+import static org.opensearch.cluster.health.ClusterHealthStatus.GREEN;
+import static org.opensearch.cluster.health.ClusterHealthStatus.RED;
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS;
 import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder;
@@ -750,6 +761,276 @@ public void testMessyElectionsStillMakeClusterGoGreen() throws Exception {
         ensureGreen("test");
     }
 
+    public void testBatchModeEnabled() throws Exception {
+        internalCluster().startClusterManagerOnlyNodes(
+            1,
+            Settings.builder().put(ExistingShardsAllocator.EXISTING_SHARDS_ALLOCATOR_BATCH_MODE.getKey(), true).build()
+        );
+        List<String> dataOnlyNodes = internalCluster().startDataOnlyNodes(2);
+        createIndex(
+            "test",
+            Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).build()
+        );
+        ensureGreen("test");
+        Settings node0DataPathSettings = internalCluster().dataPathSettings(dataOnlyNodes.get(0));
+        Settings node1DataPathSettings = internalCluster().dataPathSettings(dataOnlyNodes.get(1));
+        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(dataOnlyNodes.get(0)));
+        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(dataOnlyNodes.get(1)));
+        ensureRed("test");
+        ensureStableCluster(1);
+
+        logger.info("--> Now do a protective reroute");
+        ClusterRerouteResponse clusterRerouteResponse = client().admin().cluster().prepareReroute().setRetryFailed(true).get();
+        assertTrue(clusterRerouteResponse.isAcknowledged());
+
+        ShardsBatchGatewayAllocator gatewayAllocator = internalCluster().getInstance(
+            ShardsBatchGatewayAllocator.class,
+            internalCluster().getClusterManagerName()
+        );
+        assertTrue(ExistingShardsAllocator.EXISTING_SHARDS_ALLOCATOR_BATCH_MODE.get(internalCluster().clusterService().getSettings()));
+        assertEquals(1, gatewayAllocator.getNumberOfStartedShardBatches());
+        assertEquals(1, gatewayAllocator.getNumberOfStoreShardBatches());
+
+        // Now start both data nodes and ensure batch mode is working
+        logger.info("--> restarting the stopped nodes");
+        internalCluster().startDataOnlyNode(Settings.builder().put("node.name", dataOnlyNodes.get(0)).put(node0DataPathSettings).build());
+        internalCluster().startDataOnlyNode(Settings.builder().put("node.name", dataOnlyNodes.get(1)).put(node1DataPathSettings).build());
+        ensureStableCluster(3);
+        ensureGreen("test");
+        assertEquals(0, gatewayAllocator.getNumberOfStartedShardBatches());
+        assertEquals(0, gatewayAllocator.getNumberOfStoreShardBatches());
+        assertEquals(0, gatewayAllocator.getNumberOfInFlightFetches());
+    }
+
+    public void testBatchModeDisabled() throws Exception {
+        internalCluster().startClusterManagerOnlyNodes(
+            1,
+            Settings.builder().put(ExistingShardsAllocator.EXISTING_SHARDS_ALLOCATOR_BATCH_MODE.getKey(), false).build()
+        );
+        List<String> dataOnlyNodes = internalCluster().startDataOnlyNodes(2);
+        createIndex(
+            "test",
+            Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).build()
+        );
+
+        ensureGreen("test");
+        Settings node0DataPathSettings = internalCluster().dataPathSettings(dataOnlyNodes.get(0));
+        Settings node1DataPathSettings = internalCluster().dataPathSettings(dataOnlyNodes.get(1));
+        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(dataOnlyNodes.get(0)));
+        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(dataOnlyNodes.get(1)));
+        ensureStableCluster(1);
+
+        logger.info("--> Now do a protective reroute");
+        ClusterRerouteResponse clusterRerouteResponse = client().admin().cluster().prepareReroute().setRetryFailed(true).get();
+        assertTrue(clusterRerouteResponse.isAcknowledged());
+
+        ShardsBatchGatewayAllocator gatewayAllocator = internalCluster().getInstance(
+            ShardsBatchGatewayAllocator.class,
+            internalCluster().getClusterManagerName()
+        );
+        ensureRed("test");
+
+        assertFalse(ExistingShardsAllocator.EXISTING_SHARDS_ALLOCATOR_BATCH_MODE.get(internalCluster().clusterService().getSettings()));
+
+        // assert no batches created
+        assertEquals(0, gatewayAllocator.getNumberOfStartedShardBatches());
+        assertEquals(0, gatewayAllocator.getNumberOfStoreShardBatches());
+
+        logger.info("--> restarting the stopped nodes");
+        internalCluster().startDataOnlyNode(Settings.builder().put("node.name", dataOnlyNodes.get(0)).put(node0DataPathSettings).build());
+        internalCluster().startDataOnlyNode(Settings.builder().put("node.name", dataOnlyNodes.get(1)).put(node1DataPathSettings).build());
+        ensureStableCluster(3);
+        ensureGreen("test");
+    }
+
+    public void testNBatchesCreationAndAssignment() throws Exception {
+        // we will reduce batch size to 5 to make sure we have enough batches to test assignment
+        // Total number of primary shards = 50 (50 indices*1)
+        // Total number of replica shards = 50 (50 indices*1)
+        // Total batches creation for primaries and replicas will be 10 each
+
+        internalCluster().startClusterManagerOnlyNodes(
+            1,
+            Settings.builder().put(ExistingShardsAllocator.EXISTING_SHARDS_ALLOCATOR_BATCH_MODE.getKey(), true).build()
+        );
+        List<String> dataOnlyNodes = internalCluster().startDataOnlyNodes(2);
+        createNIndices(50, "test");
+        ensureStableCluster(3);
+        IndicesStatsResponse indicesStats = dataNodeClient().admin().indices().prepareStats().get();
+        assertThat(indicesStats.getSuccessfulShards(), equalTo(100));
+        ClusterHealthResponse health = client().admin()
+            .cluster()
+            .health(Requests.clusterHealthRequest().waitForGreenStatus().timeout("1m"))
+            .actionGet();
+        assertFalse(health.isTimedOut());
+        assertEquals(GREEN, health.getStatus());
+
+        String clusterManagerName = internalCluster().getClusterManagerName();
+        Settings clusterManagerDataPathSettings = internalCluster().dataPathSettings(clusterManagerName);
+        Settings node0DataPathSettings = internalCluster().dataPathSettings(dataOnlyNodes.get(0));
+        Settings node1DataPathSettings = internalCluster().dataPathSettings(dataOnlyNodes.get(1));
+
+        internalCluster().stopCurrentClusterManagerNode();
+        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(dataOnlyNodes.get(0)));
+        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(dataOnlyNodes.get(1)));
+
+        // Now start cluster manager node and post that verify batches created
+        internalCluster().startClusterManagerOnlyNodes(
+            1,
+            Settings.builder()
+                .put("node.name", clusterManagerName)
+                .put(clusterManagerDataPathSettings)
+                .put(ShardsBatchGatewayAllocator.GATEWAY_ALLOCATOR_BATCH_SIZE.getKey(), 5)
+                .put(ExistingShardsAllocator.EXISTING_SHARDS_ALLOCATOR_BATCH_MODE.getKey(), true)
+                .build()
+        );
+        ensureStableCluster(1);
+
+        logger.info("--> Now do a protective reroute"); // to avoid any race condition in test
+        ClusterRerouteResponse clusterRerouteResponse = client().admin().cluster().prepareReroute().setRetryFailed(true).get();
+        assertTrue(clusterRerouteResponse.isAcknowledged());
+
+        ShardsBatchGatewayAllocator gatewayAllocator = internalCluster().getInstance(
+            ShardsBatchGatewayAllocator.class,
+            internalCluster().getClusterManagerName()
+        );
+        assertTrue(ExistingShardsAllocator.EXISTING_SHARDS_ALLOCATOR_BATCH_MODE.get(internalCluster().clusterService().getSettings()));
+        assertEquals(10, gatewayAllocator.getNumberOfStartedShardBatches());
+        assertEquals(10, gatewayAllocator.getNumberOfStoreShardBatches());
+        health = client(internalCluster().getClusterManagerName()).admin().cluster().health(Requests.clusterHealthRequest()).actionGet();
+        assertFalse(health.isTimedOut());
+        assertEquals(RED, health.getStatus());
+        assertEquals(100, health.getUnassignedShards());
+        assertEquals(0, health.getInitializingShards());
+        assertEquals(0, health.getActiveShards());
+        assertEquals(0, health.getRelocatingShards());
+        assertEquals(0, health.getNumberOfDataNodes());
+
+        // Now start both data nodes and ensure batch mode is working
+        logger.info("--> restarting the stopped nodes");
+        internalCluster().startDataOnlyNode(Settings.builder().put("node.name", dataOnlyNodes.get(0)).put(node0DataPathSettings).build());
+        internalCluster().startDataOnlyNode(Settings.builder().put("node.name", dataOnlyNodes.get(1)).put(node1DataPathSettings).build());
+        ensureStableCluster(3);
+
+        // wait for cluster to turn green
+        health = client().admin().cluster().health(Requests.clusterHealthRequest().waitForGreenStatus().timeout("5m")).actionGet();
+        assertFalse(health.isTimedOut());
+        assertEquals(GREEN, health.getStatus());
+        assertEquals(0, health.getUnassignedShards());
+        assertEquals(0, health.getInitializingShards());
+        assertEquals(100, health.getActiveShards());
+        assertEquals(0, health.getRelocatingShards());
+        assertEquals(2, health.getNumberOfDataNodes());
+        assertEquals(0, gatewayAllocator.getNumberOfStartedShardBatches());
+        assertEquals(0, gatewayAllocator.getNumberOfStoreShardBatches());
+    }
+
+    public void testCulpritShardInBatch() throws Exception {
+        internalCluster().startClusterManagerOnlyNodes(
+            1,
+            Settings.builder().put(ExistingShardsAllocator.EXISTING_SHARDS_ALLOCATOR_BATCH_MODE.getKey(), true).build()
+        );
+        List<String> dataOnlyNodes = internalCluster().startDataOnlyNodes(3);
+        createNIndices(4, "test");
+        ensureStableCluster(4);
+        ClusterHealthResponse health = client().admin()
+            .cluster()
+            .health(Requests.clusterHealthRequest().waitForGreenStatus().timeout("5m"))
+            .actionGet();
+        assertFalse(health.isTimedOut());
+        assertEquals(GREEN, health.getStatus());
+        assertEquals(8, health.getActiveShards());
+
+        String culpritShardIndexName = "test0";
+        Index idx = resolveIndex(culpritShardIndexName);
+        for (String node : internalCluster().nodesInclude(culpritShardIndexName)) {
+            IndicesService indexServices = internalCluster().getInstance(IndicesService.class, node);
+            IndexService indexShards = indexServices.indexServiceSafe(idx);
+            Integer shardId = 0;
+            IndexShard shard = indexShards.getShard(0);
+            logger.debug("--> failing shard [{}] on node [{}]", shardId, node);
+            shard.failShard("test", new CorruptIndexException("test corrupted", ""));
+            logger.debug("--> failed shard [{}] on node [{}]", shardId, node);
+        }
+
+        String clusterManagerName = internalCluster().getClusterManagerName();
+        Settings clusterManagerDataPathSettings = internalCluster().dataPathSettings(clusterManagerName);
+        Settings node0DataPathSettings = internalCluster().dataPathSettings(dataOnlyNodes.get(0));
+        Settings node1DataPathSettings = internalCluster().dataPathSettings(dataOnlyNodes.get(1));
+        Settings node2DataPathSettings = internalCluster().dataPathSettings(dataOnlyNodes.get(2));
+
+        internalCluster().stopCurrentClusterManagerNode();
+        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(dataOnlyNodes.get(0)));
+        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(dataOnlyNodes.get(1)));
+        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(dataOnlyNodes.get(2)));
+
+        // Now start cluster manager node and post that verify batches created
+        internalCluster().startClusterManagerOnlyNodes(
+            1,
+            Settings.builder()
+                .put("node.name", clusterManagerName)
+                .put(clusterManagerDataPathSettings)
+                .put(ExistingShardsAllocator.EXISTING_SHARDS_ALLOCATOR_BATCH_MODE.getKey(), true)
+                .build()
+        );
+        ensureStableCluster(1);
+
+        logger.info("--> Now do a protective reroute"); // to avoid any race condition in test
+        ClusterRerouteResponse clusterRerouteResponse = client().admin().cluster().prepareReroute().setRetryFailed(true).get();
+        assertTrue(clusterRerouteResponse.isAcknowledged());
+
+        ShardsBatchGatewayAllocator gatewayAllocator = internalCluster().getInstance(
+            ShardsBatchGatewayAllocator.class,
+            internalCluster().getClusterManagerName()
+        );
+        assertTrue(ExistingShardsAllocator.EXISTING_SHARDS_ALLOCATOR_BATCH_MODE.get(internalCluster().clusterService().getSettings()));
+        assertEquals(1, gatewayAllocator.getNumberOfStartedShardBatches());
+        assertEquals(1, gatewayAllocator.getNumberOfStoreShardBatches());
+        assertTrue(clusterRerouteResponse.isAcknowledged());
+        health = client(internalCluster().getClusterManagerName()).admin().cluster().health(Requests.clusterHealthRequest()).actionGet();
+        assertFalse(health.isTimedOut());
+        assertEquals(RED, health.getStatus());
+        assertEquals(8, health.getUnassignedShards());
+        assertEquals(0, health.getInitializingShards());
+        assertEquals(0, health.getActiveShards());
+        assertEquals(0, health.getRelocatingShards());
+        assertEquals(0, health.getNumberOfDataNodes());
+
+        logger.info("--> restarting the stopped nodes");
+        internalCluster().startDataOnlyNode(Settings.builder().put("node.name", dataOnlyNodes.get(0)).put(node0DataPathSettings).build());
+        internalCluster().startDataOnlyNode(Settings.builder().put("node.name", dataOnlyNodes.get(1)).put(node1DataPathSettings).build());
+        internalCluster().startDataOnlyNode(Settings.builder().put("node.name", dataOnlyNodes.get(2)).put(node2DataPathSettings).build());
+        ensureStableCluster(4);
+
+        health = client().admin().cluster().health(Requests.clusterHealthRequest().waitForGreenStatus().timeout("1m")).actionGet();
+
+        assertEquals(RED, health.getStatus());
+        assertTrue(health.isTimedOut());
+        assertEquals(0, health.getNumberOfPendingTasks());
+        assertEquals(0, health.getNumberOfInFlightFetch());
+        assertEquals(6, health.getActiveShards());
+        assertEquals(2, health.getUnassignedShards());
+        assertEquals(0, health.getInitializingShards());
+        assertEquals(0, health.getRelocatingShards());
+        assertEquals(3, health.getNumberOfDataNodes());
+    }
+
+    private void createNIndices(int n, String prefix) {
+
+        for (int i = 0; i < n; i++) {
+            createIndex(
+                prefix + i,
+                Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).build()
+            );
+            // index doc2
+            client().prepareIndex(prefix + i).setId("1").setSource("foo", "bar").get();
+
+            // index doc 2
+            client().prepareIndex(prefix + i).setId("2").setSource("foo2", "bar2").get();
+            ensureGreen(prefix + i);
+        }
+    }
+
     public void testSingleShardFetchUsingBatchAction() {
         String indexName = "test";
         int numOfShards = 1;
@@ -909,6 +1190,56 @@ public void testShardStoreFetchCorruptedIndexUsingBatchAction() throws Exception
         assertNodeStoreFilesMetadataSuccessCase(nodeStoreFilesMetadata.get(shardId2), shardId2);
     }
 
+    public void testDeleteRedIndexInBatchMode() throws Exception {
+        internalCluster().startClusterManagerOnlyNodes(
+            1,
+            Settings.builder().put(ExistingShardsAllocator.EXISTING_SHARDS_ALLOCATOR_BATCH_MODE.getKey(), true).build()
+        );
+        List<String> dataOnlyNodes = internalCluster().startDataOnlyNodes(
+            2,
+            Settings.builder().put(ExistingShardsAllocator.EXISTING_SHARDS_ALLOCATOR_BATCH_MODE.getKey(), true).build()
+        );
+        createIndex(
+            "test",
+            Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build()
+        );
+        createIndex(
+            "test1",
+            Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build()
+        );
+        createIndex(
+            "test2",
+            Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build()
+        );
+        createIndex(
+            "testg",
+            Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).build()
+        );
+
+        ensureGreen("test", "test1", "test2", "testg");
+        internalCluster().stopRandomDataNode();
+        ensureStableCluster(2);
+
+        ShardsBatchGatewayAllocator gatewayAllocator = internalCluster().getInstance(
+            ShardsBatchGatewayAllocator.class,
+            internalCluster().getClusterManagerName()
+        );
+        ensureRed("test", "test1", "test2");
+
+        assertTrue(ExistingShardsAllocator.EXISTING_SHARDS_ALLOCATOR_BATCH_MODE.get(internalCluster().clusterService().getSettings()));
+
+        logger.info("--> Now do a reroute so batches are created"); // to avoid any race condition in test
+        ClusterRerouteResponse clusterRerouteResponse = client().admin().cluster().prepareReroute().setRetryFailed(true).get();
+        assertTrue(clusterRerouteResponse.isAcknowledged());
+
+        AcknowledgedResponse deleteIndexResponse = client().admin().indices().prepareDelete("test").get();
+        assertTrue(deleteIndexResponse.isAcknowledged());
+
+        ensureYellow("testg");
+        IndicesExistsResponse indexExistResponse = client().admin().indices().prepareExists("test").get();
+        assertFalse(indexExistResponse.isExists());
+    }
+
     private void prepareIndices(String[] indices, int numberOfPrimaryShards, int numberOfReplicaShards) {
         for (String index : indices) {
             createIndex(
diff --git a/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java b/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java
index 7e0c1630a76e4..f97950f2652a3 100644
--- a/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/index/shard/IndexShardIT.java
@@ -84,6 +84,7 @@
 import org.opensearch.index.translog.TestTranslog;
 import org.opensearch.index.translog.Translog;
 import org.opensearch.index.translog.TranslogStats;
+import org.opensearch.indices.DefaultRemoteStoreSettings;
 import org.opensearch.indices.IndicesService;
 import org.opensearch.indices.recovery.RecoveryState;
 import org.opensearch.indices.replication.checkpoint.SegmentReplicationCheckpointPublisher;
@@ -711,10 +712,11 @@ public static final IndexShard newIndexShard(
             SegmentReplicationCheckpointPublisher.EMPTY,
             null,
             null,
-            () -> IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL,
             nodeId,
             null,
-            false
+            DefaultRemoteStoreSettings.INSTANCE,
+            false,
+            IndexShardTestUtils.getFakeDiscoveryNodes(initializingShardRouting)
         );
     }
 
diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/CacheStatsAPIIndicesRequestCacheIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/CacheStatsAPIIndicesRequestCacheIT.java
new file mode 100644
index 0000000000000..de7a52761c77c
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/CacheStatsAPIIndicesRequestCacheIT.java
@@ -0,0 +1,291 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.indices;
+
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+
+import org.opensearch.action.admin.cluster.node.stats.NodesStatsRequest;
+import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse;
+import org.opensearch.action.admin.indices.stats.CommonStatsFlags;
+import org.opensearch.action.search.SearchResponse;
+import org.opensearch.client.Client;
+import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.common.Randomness;
+import org.opensearch.common.cache.CacheType;
+import org.opensearch.common.cache.service.NodeCacheStats;
+import org.opensearch.common.cache.stats.ImmutableCacheStats;
+import org.opensearch.common.cache.stats.ImmutableCacheStatsHolderTests;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.FeatureFlags;
+import org.opensearch.common.xcontent.XContentFactory;
+import org.opensearch.common.xcontent.XContentHelper;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
+import org.opensearch.core.xcontent.ToXContent;
+import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.index.cache.request.RequestCacheStats;
+import org.opensearch.index.query.QueryBuilders;
+import org.opensearch.test.OpenSearchIntegTestCase;
+import org.opensearch.test.ParameterizedStaticSettingsOpenSearchIntegTestCase;
+import org.opensearch.test.hamcrest.OpenSearchAssertions;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse;
+
+// Use a single data node to simplify logic about cache stats across different shards.
+@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 1)
+public class CacheStatsAPIIndicesRequestCacheIT extends ParameterizedStaticSettingsOpenSearchIntegTestCase {
+    public CacheStatsAPIIndicesRequestCacheIT(Settings settings) {
+        super(settings);
+    }
+
+    @ParametersFactory
+    public static Collection<Object[]> parameters() {
+        return Arrays.<Object[]>asList(new Object[] { Settings.builder().put(FeatureFlags.PLUGGABLE_CACHE, "true").build() });
+    }
+
+    public void testCacheStatsAPIWIthOnHeapCache() throws Exception {
+        String index1Name = "index1";
+        String index2Name = "index2";
+        Client client = client();
+
+        startIndex(client, index1Name);
+        startIndex(client, index2Name);
+
+        // Search twice for the same doc in index 1
+        for (int i = 0; i < 2; i++) {
+            searchIndex(client, index1Name, "");
+        }
+
+        // Search once for a doc in index 2
+        searchIndex(client, index2Name, "");
+
+        // First, aggregate by indices only
+        Map<String, Object> xContentMap = getNodeCacheStatsXContentMap(client, List.of(IndicesRequestCache.INDEX_DIMENSION_NAME));
+
+        List<String> index1Keys = List.of(CacheType.INDICES_REQUEST_CACHE.getValue(), IndicesRequestCache.INDEX_DIMENSION_NAME, index1Name);
+        // Since we searched twice, we expect to see 1 hit, 1 miss and 1 entry for index 1
+        ImmutableCacheStats expectedStats = new ImmutableCacheStats(1, 1, 0, 0, 1);
+        checkCacheStatsAPIResponse(xContentMap, index1Keys, expectedStats, false, true);
+        // Get the request size for one request, so we can reuse it for next index
+        int requestSize = (int) ((Map<String, Object>) ImmutableCacheStatsHolderTests.getValueFromNestedXContentMap(
+            xContentMap,
+            index1Keys
+        )).get(ImmutableCacheStats.Fields.SIZE_IN_BYTES);
+        assertTrue(requestSize > 0);
+
+        List<String> index2Keys = List.of(CacheType.INDICES_REQUEST_CACHE.getValue(), IndicesRequestCache.INDEX_DIMENSION_NAME, index2Name);
+        // We searched once in index 2, we expect 1 miss + 1 entry
+        expectedStats = new ImmutableCacheStats(0, 1, 0, requestSize, 1);
+        checkCacheStatsAPIResponse(xContentMap, index2Keys, expectedStats, true, true);
+
+        // The total stats for the node should be 1 hit, 2 misses, and 2 entries
+        expectedStats = new ImmutableCacheStats(1, 2, 0, 2 * requestSize, 2);
+        List<String> totalStatsKeys = List.of(CacheType.INDICES_REQUEST_CACHE.getValue());
+        checkCacheStatsAPIResponse(xContentMap, totalStatsKeys, expectedStats, true, true);
+
+        // Aggregate by shards only
+        xContentMap = getNodeCacheStatsXContentMap(client, List.of(IndicesRequestCache.SHARD_ID_DIMENSION_NAME));
+
+        List<String> index1Shard0Keys = List.of(
+            CacheType.INDICES_REQUEST_CACHE.getValue(),
+            IndicesRequestCache.SHARD_ID_DIMENSION_NAME,
+            "[" + index1Name + "][0]"
+        );
+
+        expectedStats = new ImmutableCacheStats(1, 1, 0, requestSize, 1);
+        checkCacheStatsAPIResponse(xContentMap, index1Shard0Keys, expectedStats, true, true);
+
+        List<String> index2Shard0Keys = List.of(
+            CacheType.INDICES_REQUEST_CACHE.getValue(),
+            IndicesRequestCache.SHARD_ID_DIMENSION_NAME,
+            "[" + index2Name + "][0]"
+        );
+        expectedStats = new ImmutableCacheStats(0, 1, 0, requestSize, 1);
+        checkCacheStatsAPIResponse(xContentMap, index2Shard0Keys, expectedStats, true, true);
+
+        // Aggregate by indices and shards
+        xContentMap = getNodeCacheStatsXContentMap(
+            client,
+            List.of(IndicesRequestCache.INDEX_DIMENSION_NAME, IndicesRequestCache.SHARD_ID_DIMENSION_NAME)
+        );
+
+        index1Keys = List.of(
+            CacheType.INDICES_REQUEST_CACHE.getValue(),
+            IndicesRequestCache.INDEX_DIMENSION_NAME,
+            index1Name,
+            IndicesRequestCache.SHARD_ID_DIMENSION_NAME,
+            "[" + index1Name + "][0]"
+        );
+
+        expectedStats = new ImmutableCacheStats(1, 1, 0, requestSize, 1);
+        checkCacheStatsAPIResponse(xContentMap, index1Keys, expectedStats, true, true);
+
+        index2Keys = List.of(
+            CacheType.INDICES_REQUEST_CACHE.getValue(),
+            IndicesRequestCache.INDEX_DIMENSION_NAME,
+            index2Name,
+            IndicesRequestCache.SHARD_ID_DIMENSION_NAME,
+            "[" + index2Name + "][0]"
+        );
+
+        expectedStats = new ImmutableCacheStats(0, 1, 0, requestSize, 1);
+        checkCacheStatsAPIResponse(xContentMap, index2Keys, expectedStats, true, true);
+
+    }
+
+    // TODO: Add testCacheStatsAPIWithTieredCache when TSC stats implementation PR is merged
+
+    public void testStatsMatchOldApi() throws Exception {
+        // The main purpose of this test is to check that the new and old APIs are both correctly estimating memory size,
+        // using the logic that includes the overhead memory in ICacheKey.
+        String index = "index";
+        Client client = client();
+        startIndex(client, index);
+
+        int numKeys = Randomness.get().nextInt(100) + 1;
+        for (int i = 0; i < numKeys; i++) {
+            searchIndex(client, index, String.valueOf(i));
+        }
+        // Get some hits as well
+        for (int i = 0; i < numKeys / 2; i++) {
+            searchIndex(client, index, String.valueOf(i));
+        }
+
+        RequestCacheStats oldApiStats = client.admin()
+            .indices()
+            .prepareStats(index)
+            .setRequestCache(true)
+            .get()
+            .getTotal()
+            .getRequestCache();
+        assertNotEquals(0, oldApiStats.getMemorySizeInBytes());
+
+        List<String> xContentMapKeys = List.of(CacheType.INDICES_REQUEST_CACHE.getValue());
+        Map<String, Object> xContentMap = getNodeCacheStatsXContentMap(client, List.of());
+        ImmutableCacheStats expected = new ImmutableCacheStats(
+            oldApiStats.getHitCount(),
+            oldApiStats.getMissCount(),
+            oldApiStats.getEvictions(),
+            oldApiStats.getMemorySizeInBytes(),
+            0
+        );
+        // Don't check entries, as the old API doesn't track this
+        checkCacheStatsAPIResponse(xContentMap, xContentMapKeys, expected, true, false);
+    }
+
+    public void testNullLevels() throws Exception {
+        String index = "index";
+        Client client = client();
+        startIndex(client, index);
+        int numKeys = Randomness.get().nextInt(100) + 1;
+        for (int i = 0; i < numKeys; i++) {
+            searchIndex(client, index, String.valueOf(i));
+        }
+        Map<String, Object> xContentMap = getNodeCacheStatsXContentMap(client, null);
+        // Null levels should result in only the total cache stats being returned -> 6 fields inside the response.
+        assertEquals(6, ((Map<String, Object>) xContentMap.get("request_cache")).size());
+    }
+
+    private void startIndex(Client client, String indexName) throws InterruptedException {
+        assertAcked(
+            client.admin()
+                .indices()
+                .prepareCreate(indexName)
+                .setMapping("k", "type=keyword")
+                .setSettings(
+                    Settings.builder()
+                        .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
+                        .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                        .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                )
+                .get()
+        );
+        indexRandom(true, client.prepareIndex(indexName).setSource("k", "hello"));
+        ensureSearchable(indexName);
+    }
+
+    private SearchResponse searchIndex(Client client, String index, String searchSuffix) {
+        SearchResponse resp = client.prepareSearch(index)
+            .setRequestCache(true)
+            .setQuery(QueryBuilders.termQuery("k", "hello" + searchSuffix))
+            .get();
+        assertSearchResponse(resp);
+        OpenSearchAssertions.assertAllSuccessful(resp);
+        return resp;
+    }
+
+    private static Map<String, Object> getNodeCacheStatsXContentMap(Client client, List<String> aggregationLevels) throws IOException {
+
+        CommonStatsFlags statsFlags = new CommonStatsFlags();
+        statsFlags.includeAllCacheTypes();
+        String[] flagsLevels;
+        if (aggregationLevels == null) {
+            flagsLevels = null;
+        } else {
+            flagsLevels = aggregationLevels.toArray(new String[0]);
+        }
+        statsFlags.setLevels(flagsLevels);
+
+        NodesStatsResponse nodeStatsResponse = client.admin()
+            .cluster()
+            .prepareNodesStats("data:true")
+            .addMetric(NodesStatsRequest.Metric.CACHE_STATS.metricName())
+            .setIndices(statsFlags)
+            .get();
+        // Can always get the first data node as there's only one in this test suite
+        assertEquals(1, nodeStatsResponse.getNodes().size());
+        NodeCacheStats ncs = nodeStatsResponse.getNodes().get(0).getNodeCacheStats();
+
+        XContentBuilder builder = XContentFactory.jsonBuilder();
+        Map<String, String> paramMap = new HashMap<>();
+        if (aggregationLevels != null && !aggregationLevels.isEmpty()) {
+            paramMap.put("level", String.join(",", aggregationLevels));
+        }
+        ToXContent.Params params = new ToXContent.MapParams(paramMap);
+
+        builder.startObject();
+        ncs.toXContent(builder, params);
+        builder.endObject();
+
+        String resultString = builder.toString();
+        return XContentHelper.convertToMap(MediaTypeRegistry.JSON.xContent(), resultString, true);
+    }
+
+    private static void checkCacheStatsAPIResponse(
+        Map<String, Object> xContentMap,
+        List<String> xContentMapKeys,
+        ImmutableCacheStats expectedStats,
+        boolean checkMemorySize,
+        boolean checkEntries
+    ) {
+        // Assumes the keys point to a level whose keys are the field values ("size_in_bytes", "evictions", etc) and whose values store
+        // those stats
+        Map<String, Object> aggregatedStatsResponse = (Map<String, Object>) ImmutableCacheStatsHolderTests.getValueFromNestedXContentMap(
+            xContentMap,
+            xContentMapKeys
+        );
+        assertNotNull(aggregatedStatsResponse);
+        assertEquals(expectedStats.getHits(), (int) aggregatedStatsResponse.get(ImmutableCacheStats.Fields.HIT_COUNT));
+        assertEquals(expectedStats.getMisses(), (int) aggregatedStatsResponse.get(ImmutableCacheStats.Fields.MISS_COUNT));
+        assertEquals(expectedStats.getEvictions(), (int) aggregatedStatsResponse.get(ImmutableCacheStats.Fields.EVICTIONS));
+        if (checkMemorySize) {
+            assertEquals(expectedStats.getSizeInBytes(), (int) aggregatedStatsResponse.get(ImmutableCacheStats.Fields.SIZE_IN_BYTES));
+        }
+        if (checkEntries) {
+            assertEquals(expectedStats.getItems(), (int) aggregatedStatsResponse.get(ImmutableCacheStats.Fields.ITEM_COUNT));
+        }
+    }
+}
diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
index 52b4dad553180..ae2295cb874f5 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/IndicesRequestCacheIT.java
@@ -34,24 +34,40 @@
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
+import org.opensearch.action.admin.cluster.health.ClusterHealthResponse;
+import org.opensearch.action.admin.cluster.node.stats.NodeStats;
+import org.opensearch.action.admin.cluster.node.stats.NodesStatsResponse;
+import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
 import org.opensearch.action.admin.indices.alias.Alias;
+import org.opensearch.action.admin.indices.cache.clear.ClearIndicesCacheRequest;
 import org.opensearch.action.admin.indices.forcemerge.ForceMergeResponse;
 import org.opensearch.action.search.SearchResponse;
 import org.opensearch.action.search.SearchType;
 import org.opensearch.client.Client;
+import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.cluster.routing.allocation.command.MoveAllocationCommand;
+import org.opensearch.cluster.routing.allocation.decider.EnableAllocationDecider;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.time.DateFormatter;
+import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.util.FeatureFlags;
+import org.opensearch.core.index.Index;
+import org.opensearch.core.index.shard.ShardId;
+import org.opensearch.env.NodeEnvironment;
+import org.opensearch.index.IndexNotFoundException;
 import org.opensearch.index.cache.request.RequestCacheStats;
 import org.opensearch.index.query.QueryBuilders;
 import org.opensearch.search.aggregations.bucket.global.GlobalAggregationBuilder;
 import org.opensearch.search.aggregations.bucket.histogram.DateHistogramInterval;
 import org.opensearch.search.aggregations.bucket.histogram.Histogram;
 import org.opensearch.search.aggregations.bucket.histogram.Histogram.Bucket;
+import org.opensearch.test.OpenSearchIntegTestCase;
 import org.opensearch.test.ParameterizedStaticSettingsOpenSearchIntegTestCase;
 import org.opensearch.test.hamcrest.OpenSearchAssertions;
 
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.time.ZoneId;
 import java.time.ZoneOffset;
 import java.time.ZonedDateTime;
@@ -59,7 +75,13 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.List;
+import java.util.concurrent.TimeUnit;
 
+import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
+import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS;
+import static org.opensearch.cluster.routing.allocation.decider.EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING;
+import static org.opensearch.indices.IndicesRequestCache.INDICES_REQUEST_CACHE_STALENESS_THRESHOLD_SETTING;
+import static org.opensearch.indices.IndicesService.INDICES_CACHE_CLEANUP_INTERVAL_SETTING_KEY;
 import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING;
 import static org.opensearch.search.aggregations.AggregationBuilders.dateHistogram;
 import static org.opensearch.search.aggregations.AggregationBuilders.dateRange;
@@ -69,6 +91,7 @@
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 
+@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0, supportsDedicatedMasters = false)
 public class IndicesRequestCacheIT extends ParameterizedStaticSettingsOpenSearchIntegTestCase {
     public IndicesRequestCacheIT(Settings settings) {
         super(settings);
@@ -92,25 +115,31 @@ protected boolean useRandomReplicationStrategy() {
     // One of the primary purposes of the query cache is to cache aggs results
     public void testCacheAggs() throws Exception {
         Client client = client();
+        String index = "index";
         assertAcked(
             client.admin()
                 .indices()
-                .prepareCreate("index")
+                .prepareCreate(index)
                 .setMapping("f", "type=date")
-                .setSettings(Settings.builder().put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true))
+                .setSettings(
+                    Settings.builder()
+                        .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
+                        .put(SETTING_NUMBER_OF_SHARDS, 1)
+                        .put(SETTING_NUMBER_OF_REPLICAS, 0)
+                )
                 .get()
         );
         indexRandom(
             true,
-            client.prepareIndex("index").setSource("f", "2014-03-10T00:00:00.000Z"),
-            client.prepareIndex("index").setSource("f", "2014-05-13T00:00:00.000Z")
+            client.prepareIndex(index).setSource("f", "2014-03-10T00:00:00.000Z"),
+            client.prepareIndex(index).setSource("f", "2014-05-13T00:00:00.000Z")
         );
-        ensureSearchable("index");
+        ensureSearchable(index);
 
         // This is not a random example: serialization with time zones writes shared strings
         // which used to not work well with the query cache because of the handles stream output
         // see #9500
-        final SearchResponse r1 = client.prepareSearch("index")
+        final SearchResponse r1 = client.prepareSearch(index)
             .setSize(0)
             .setSearchType(SearchType.QUERY_THEN_FETCH)
             .addAggregation(
@@ -124,12 +153,12 @@ public void testCacheAggs() throws Exception {
 
         // The cached is actually used
         assertThat(
-            client.admin().indices().prepareStats("index").setRequestCache(true).get().getTotal().getRequestCache().getMemorySizeInBytes(),
+            client.admin().indices().prepareStats(index).setRequestCache(true).get().getTotal().getRequestCache().getMemorySizeInBytes(),
             greaterThan(0L)
         );
 
         for (int i = 0; i < 10; ++i) {
-            final SearchResponse r2 = client.prepareSearch("index")
+            final SearchResponse r2 = client.prepareSearch(index)
                 .setSize(0)
                 .setSearchType(SearchType.QUERY_THEN_FETCH)
                 .addAggregation(
@@ -156,10 +185,11 @@ public void testCacheAggs() throws Exception {
 
     public void testQueryRewrite() throws Exception {
         Client client = client();
+        String index = "index";
         assertAcked(
             client.admin()
                 .indices()
-                .prepareCreate("index")
+                .prepareCreate(index)
                 .setMapping("s", "type=date")
                 .setSettings(
                     Settings.builder()
@@ -172,28 +202,28 @@ public void testQueryRewrite() throws Exception {
         );
         indexRandom(
             true,
-            client.prepareIndex("index").setId("1").setRouting("1").setSource("s", "2016-03-19"),
-            client.prepareIndex("index").setId("2").setRouting("1").setSource("s", "2016-03-20"),
-            client.prepareIndex("index").setId("3").setRouting("1").setSource("s", "2016-03-21"),
-            client.prepareIndex("index").setId("4").setRouting("2").setSource("s", "2016-03-22"),
-            client.prepareIndex("index").setId("5").setRouting("2").setSource("s", "2016-03-23"),
-            client.prepareIndex("index").setId("6").setRouting("2").setSource("s", "2016-03-24"),
-            client.prepareIndex("index").setId("7").setRouting("3").setSource("s", "2016-03-25"),
-            client.prepareIndex("index").setId("8").setRouting("3").setSource("s", "2016-03-26"),
-            client.prepareIndex("index").setId("9").setRouting("3").setSource("s", "2016-03-27")
+            client.prepareIndex(index).setId("1").setRouting("1").setSource("s", "2016-03-19"),
+            client.prepareIndex(index).setId("2").setRouting("1").setSource("s", "2016-03-20"),
+            client.prepareIndex(index).setId("3").setRouting("1").setSource("s", "2016-03-21"),
+            client.prepareIndex(index).setId("4").setRouting("2").setSource("s", "2016-03-22"),
+            client.prepareIndex(index).setId("5").setRouting("2").setSource("s", "2016-03-23"),
+            client.prepareIndex(index).setId("6").setRouting("2").setSource("s", "2016-03-24"),
+            client.prepareIndex(index).setId("7").setRouting("3").setSource("s", "2016-03-25"),
+            client.prepareIndex(index).setId("8").setRouting("3").setSource("s", "2016-03-26"),
+            client.prepareIndex(index).setId("9").setRouting("3").setSource("s", "2016-03-27")
         );
-        ensureSearchable("index");
-        assertCacheState(client, "index", 0, 0);
+        ensureSearchable(index);
+        assertCacheState(client, index, 0, 0);
 
         // Force merge the index to ensure there can be no background merges during the subsequent searches that would invalidate the cache
-        ForceMergeResponse forceMergeResponse = client.admin().indices().prepareForceMerge("index").setFlush(true).get();
+        ForceMergeResponse forceMergeResponse = client.admin().indices().prepareForceMerge(index).setFlush(true).get();
         OpenSearchAssertions.assertAllSuccessful(forceMergeResponse);
         refreshAndWaitForReplication();
-        ensureSearchable("index");
+        ensureSearchable(index);
 
-        assertCacheState(client, "index", 0, 0);
+        assertCacheState(client, index, 0, 0);
 
-        final SearchResponse r1 = client.prepareSearch("index")
+        final SearchResponse r1 = client.prepareSearch(index)
             .setSearchType(SearchType.QUERY_THEN_FETCH)
             .setSize(0)
             .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-19").lte("2016-03-25"))
@@ -202,9 +232,9 @@ public void testQueryRewrite() throws Exception {
             .get();
         OpenSearchAssertions.assertAllSuccessful(r1);
         assertThat(r1.getHits().getTotalHits().value, equalTo(7L));
-        assertCacheState(client, "index", 0, 5);
+        assertCacheState(client, index, 0, 5);
 
-        final SearchResponse r2 = client.prepareSearch("index")
+        final SearchResponse r2 = client.prepareSearch(index)
             .setSearchType(SearchType.QUERY_THEN_FETCH)
             .setSize(0)
             .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-20").lte("2016-03-26"))
@@ -212,9 +242,9 @@ public void testQueryRewrite() throws Exception {
             .get();
         OpenSearchAssertions.assertAllSuccessful(r2);
         assertThat(r2.getHits().getTotalHits().value, equalTo(7L));
-        assertCacheState(client, "index", 3, 7);
+        assertCacheState(client, index, 3, 7);
 
-        final SearchResponse r3 = client.prepareSearch("index")
+        final SearchResponse r3 = client.prepareSearch(index)
             .setSearchType(SearchType.QUERY_THEN_FETCH)
             .setSize(0)
             .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-21").lte("2016-03-27"))
@@ -222,15 +252,16 @@ public void testQueryRewrite() throws Exception {
             .get();
         OpenSearchAssertions.assertAllSuccessful(r3);
         assertThat(r3.getHits().getTotalHits().value, equalTo(7L));
-        assertCacheState(client, "index", 6, 9);
+        assertCacheState(client, index, 6, 9);
     }
 
     public void testQueryRewriteMissingValues() throws Exception {
         Client client = client();
+        String index = "index";
         assertAcked(
             client.admin()
                 .indices()
-                .prepareCreate("index")
+                .prepareCreate(index)
                 .setMapping("s", "type=date")
                 .setSettings(
                     Settings.builder()
@@ -242,61 +273,62 @@ public void testQueryRewriteMissingValues() throws Exception {
         );
         indexRandom(
             true,
-            client.prepareIndex("index").setId("1").setSource("s", "2016-03-19"),
-            client.prepareIndex("index").setId("2").setSource("s", "2016-03-20"),
-            client.prepareIndex("index").setId("3").setSource("s", "2016-03-21"),
-            client.prepareIndex("index").setId("4").setSource("s", "2016-03-22"),
-            client.prepareIndex("index").setId("5").setSource("s", "2016-03-23"),
-            client.prepareIndex("index").setId("6").setSource("s", "2016-03-24"),
-            client.prepareIndex("index").setId("7").setSource("other", "value"),
-            client.prepareIndex("index").setId("8").setSource("s", "2016-03-26"),
-            client.prepareIndex("index").setId("9").setSource("s", "2016-03-27")
+            client.prepareIndex(index).setId("1").setSource("s", "2016-03-19"),
+            client.prepareIndex(index).setId("2").setSource("s", "2016-03-20"),
+            client.prepareIndex(index).setId("3").setSource("s", "2016-03-21"),
+            client.prepareIndex(index).setId("4").setSource("s", "2016-03-22"),
+            client.prepareIndex(index).setId("5").setSource("s", "2016-03-23"),
+            client.prepareIndex(index).setId("6").setSource("s", "2016-03-24"),
+            client.prepareIndex(index).setId("7").setSource("other", "value"),
+            client.prepareIndex(index).setId("8").setSource("s", "2016-03-26"),
+            client.prepareIndex(index).setId("9").setSource("s", "2016-03-27")
         );
-        ensureSearchable("index");
-        assertCacheState(client, "index", 0, 0);
+        ensureSearchable(index);
+        assertCacheState(client, index, 0, 0);
 
         // Force merge the index to ensure there can be no background merges during the subsequent searches that would invalidate the cache
-        ForceMergeResponse forceMergeResponse = client.admin().indices().prepareForceMerge("index").setFlush(true).get();
+        ForceMergeResponse forceMergeResponse = client.admin().indices().prepareForceMerge(index).setFlush(true).get();
         OpenSearchAssertions.assertAllSuccessful(forceMergeResponse);
         refreshAndWaitForReplication();
-        ensureSearchable("index");
+        ensureSearchable(index);
 
-        assertCacheState(client, "index", 0, 0);
+        assertCacheState(client, index, 0, 0);
 
-        final SearchResponse r1 = client.prepareSearch("index")
+        final SearchResponse r1 = client.prepareSearch(index)
             .setSearchType(SearchType.QUERY_THEN_FETCH)
             .setSize(0)
             .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-19").lte("2016-03-28"))
             .get();
         OpenSearchAssertions.assertAllSuccessful(r1);
         assertThat(r1.getHits().getTotalHits().value, equalTo(8L));
-        assertCacheState(client, "index", 0, 1);
+        assertCacheState(client, index, 0, 1);
 
-        final SearchResponse r2 = client.prepareSearch("index")
+        final SearchResponse r2 = client.prepareSearch(index)
             .setSearchType(SearchType.QUERY_THEN_FETCH)
             .setSize(0)
             .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-19").lte("2016-03-28"))
             .get();
         OpenSearchAssertions.assertAllSuccessful(r2);
         assertThat(r2.getHits().getTotalHits().value, equalTo(8L));
-        assertCacheState(client, "index", 1, 1);
+        assertCacheState(client, index, 1, 1);
 
-        final SearchResponse r3 = client.prepareSearch("index")
+        final SearchResponse r3 = client.prepareSearch(index)
             .setSearchType(SearchType.QUERY_THEN_FETCH)
             .setSize(0)
             .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-19").lte("2016-03-28"))
             .get();
         OpenSearchAssertions.assertAllSuccessful(r3);
         assertThat(r3.getHits().getTotalHits().value, equalTo(8L));
-        assertCacheState(client, "index", 2, 1);
+        assertCacheState(client, index, 2, 1);
     }
 
     public void testQueryRewriteDates() throws Exception {
         Client client = client();
+        String index = "index";
         assertAcked(
             client.admin()
                 .indices()
-                .prepareCreate("index")
+                .prepareCreate(index)
                 .setMapping("d", "type=date")
                 .setSettings(
                     Settings.builder()
@@ -308,28 +340,28 @@ public void testQueryRewriteDates() throws Exception {
         );
         indexRandom(
             true,
-            client.prepareIndex("index").setId("1").setSource("d", "2014-01-01T00:00:00"),
-            client.prepareIndex("index").setId("2").setSource("d", "2014-02-01T00:00:00"),
-            client.prepareIndex("index").setId("3").setSource("d", "2014-03-01T00:00:00"),
-            client.prepareIndex("index").setId("4").setSource("d", "2014-04-01T00:00:00"),
-            client.prepareIndex("index").setId("5").setSource("d", "2014-05-01T00:00:00"),
-            client.prepareIndex("index").setId("6").setSource("d", "2014-06-01T00:00:00"),
-            client.prepareIndex("index").setId("7").setSource("d", "2014-07-01T00:00:00"),
-            client.prepareIndex("index").setId("8").setSource("d", "2014-08-01T00:00:00"),
-            client.prepareIndex("index").setId("9").setSource("d", "2014-09-01T00:00:00")
+            client.prepareIndex(index).setId("1").setSource("d", "2014-01-01T00:00:00"),
+            client.prepareIndex(index).setId("2").setSource("d", "2014-02-01T00:00:00"),
+            client.prepareIndex(index).setId("3").setSource("d", "2014-03-01T00:00:00"),
+            client.prepareIndex(index).setId("4").setSource("d", "2014-04-01T00:00:00"),
+            client.prepareIndex(index).setId("5").setSource("d", "2014-05-01T00:00:00"),
+            client.prepareIndex(index).setId("6").setSource("d", "2014-06-01T00:00:00"),
+            client.prepareIndex(index).setId("7").setSource("d", "2014-07-01T00:00:00"),
+            client.prepareIndex(index).setId("8").setSource("d", "2014-08-01T00:00:00"),
+            client.prepareIndex(index).setId("9").setSource("d", "2014-09-01T00:00:00")
         );
-        ensureSearchable("index");
-        assertCacheState(client, "index", 0, 0);
+        ensureSearchable(index);
+        assertCacheState(client, index, 0, 0);
 
         // Force merge the index to ensure there can be no background merges during the subsequent searches that would invalidate the cache
-        ForceMergeResponse forceMergeResponse = client.admin().indices().prepareForceMerge("index").setFlush(true).get();
+        ForceMergeResponse forceMergeResponse = client.admin().indices().prepareForceMerge(index).setFlush(true).get();
         OpenSearchAssertions.assertAllSuccessful(forceMergeResponse);
         refreshAndWaitForReplication();
-        ensureSearchable("index");
+        ensureSearchable(index);
 
-        assertCacheState(client, "index", 0, 0);
+        assertCacheState(client, index, 0, 0);
 
-        final SearchResponse r1 = client.prepareSearch("index")
+        final SearchResponse r1 = client.prepareSearch(index)
             .setSearchType(SearchType.QUERY_THEN_FETCH)
             .setSize(0)
             .setQuery(QueryBuilders.rangeQuery("d").gte("2013-01-01T00:00:00").lte("now"))
@@ -338,9 +370,9 @@ public void testQueryRewriteDates() throws Exception {
             .get();
         OpenSearchAssertions.assertAllSuccessful(r1);
         assertThat(r1.getHits().getTotalHits().value, equalTo(9L));
-        assertCacheState(client, "index", 0, 1);
+        assertCacheState(client, index, 0, 1);
 
-        final SearchResponse r2 = client.prepareSearch("index")
+        final SearchResponse r2 = client.prepareSearch(index)
             .setSearchType(SearchType.QUERY_THEN_FETCH)
             .setSize(0)
             .setQuery(QueryBuilders.rangeQuery("d").gte("2013-01-01T00:00:00").lte("now"))
@@ -348,9 +380,9 @@ public void testQueryRewriteDates() throws Exception {
             .get();
         OpenSearchAssertions.assertAllSuccessful(r2);
         assertThat(r2.getHits().getTotalHits().value, equalTo(9L));
-        assertCacheState(client, "index", 1, 1);
+        assertCacheState(client, index, 1, 1);
 
-        final SearchResponse r3 = client.prepareSearch("index")
+        final SearchResponse r3 = client.prepareSearch(index)
             .setSearchType(SearchType.QUERY_THEN_FETCH)
             .setSize(0)
             .setQuery(QueryBuilders.rangeQuery("d").gte("2013-01-01T00:00:00").lte("now"))
@@ -358,7 +390,7 @@ public void testQueryRewriteDates() throws Exception {
             .get();
         OpenSearchAssertions.assertAllSuccessful(r3);
         assertThat(r3.getHits().getTotalHits().value, equalTo(9L));
-        assertCacheState(client, "index", 2, 1);
+        assertCacheState(client, index, 2, 1);
     }
 
     public void testQueryRewriteDatesWithNow() throws Exception {
@@ -449,53 +481,54 @@ public void testCanCache() throws Exception {
             .put("index.number_of_routing_shards", 2)
             .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
             .build();
-        assertAcked(client.admin().indices().prepareCreate("index").setMapping("s", "type=date").setSettings(settings).get());
+        String index = "index";
+        assertAcked(client.admin().indices().prepareCreate(index).setMapping("s", "type=date").setSettings(settings).get());
         indexRandom(
             true,
-            client.prepareIndex("index").setId("1").setRouting("1").setSource("s", "2016-03-19"),
-            client.prepareIndex("index").setId("2").setRouting("1").setSource("s", "2016-03-20"),
-            client.prepareIndex("index").setId("3").setRouting("1").setSource("s", "2016-03-21"),
-            client.prepareIndex("index").setId("4").setRouting("2").setSource("s", "2016-03-22"),
-            client.prepareIndex("index").setId("5").setRouting("2").setSource("s", "2016-03-23"),
-            client.prepareIndex("index").setId("6").setRouting("2").setSource("s", "2016-03-24"),
-            client.prepareIndex("index").setId("7").setRouting("3").setSource("s", "2016-03-25"),
-            client.prepareIndex("index").setId("8").setRouting("3").setSource("s", "2016-03-26"),
-            client.prepareIndex("index").setId("9").setRouting("3").setSource("s", "2016-03-27")
+            client.prepareIndex(index).setId("1").setRouting("1").setSource("s", "2016-03-19"),
+            client.prepareIndex(index).setId("2").setRouting("1").setSource("s", "2016-03-20"),
+            client.prepareIndex(index).setId("3").setRouting("1").setSource("s", "2016-03-21"),
+            client.prepareIndex(index).setId("4").setRouting("2").setSource("s", "2016-03-22"),
+            client.prepareIndex(index).setId("5").setRouting("2").setSource("s", "2016-03-23"),
+            client.prepareIndex(index).setId("6").setRouting("2").setSource("s", "2016-03-24"),
+            client.prepareIndex(index).setId("7").setRouting("3").setSource("s", "2016-03-25"),
+            client.prepareIndex(index).setId("8").setRouting("3").setSource("s", "2016-03-26"),
+            client.prepareIndex(index).setId("9").setRouting("3").setSource("s", "2016-03-27")
         );
-        ensureSearchable("index");
-        assertCacheState(client, "index", 0, 0);
+        ensureSearchable(index);
+        assertCacheState(client, index, 0, 0);
 
         // Force merge the index to ensure there can be no background merges during the subsequent searches that would invalidate the cache
-        ForceMergeResponse forceMergeResponse = client.admin().indices().prepareForceMerge("index").setFlush(true).get();
+        ForceMergeResponse forceMergeResponse = client.admin().indices().prepareForceMerge(index).setFlush(true).get();
         OpenSearchAssertions.assertAllSuccessful(forceMergeResponse);
         refreshAndWaitForReplication();
-        ensureSearchable("index");
+        ensureSearchable(index);
 
-        assertCacheState(client, "index", 0, 0);
+        assertCacheState(client, index, 0, 0);
 
         // If size > 0 we should no cache by default
-        final SearchResponse r1 = client.prepareSearch("index")
+        final SearchResponse r1 = client.prepareSearch(index)
             .setSearchType(SearchType.QUERY_THEN_FETCH)
             .setSize(1)
             .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-19").lte("2016-03-25"))
             .get();
         OpenSearchAssertions.assertAllSuccessful(r1);
         assertThat(r1.getHits().getTotalHits().value, equalTo(7L));
-        assertCacheState(client, "index", 0, 0);
+        assertCacheState(client, index, 0, 0);
 
         // If search type is DFS_QUERY_THEN_FETCH we should not cache
-        final SearchResponse r2 = client.prepareSearch("index")
+        final SearchResponse r2 = client.prepareSearch(index)
             .setSearchType(SearchType.DFS_QUERY_THEN_FETCH)
             .setSize(0)
             .setQuery(QueryBuilders.rangeQuery("s").gte("2016-03-20").lte("2016-03-26"))
             .get();
         OpenSearchAssertions.assertAllSuccessful(r2);
         assertThat(r2.getHits().getTotalHits().value, equalTo(7L));
-        assertCacheState(client, "index", 0, 0);
+        assertCacheState(client, index, 0, 0);
 
         // If search type is DFS_QUERY_THEN_FETCH we should not cache even if
         // the cache flag is explicitly set on the request
-        final SearchResponse r3 = client.prepareSearch("index")
+        final SearchResponse r3 = client.prepareSearch(index)
             .setSearchType(SearchType.DFS_QUERY_THEN_FETCH)
             .setSize(0)
             .setRequestCache(true)
@@ -503,10 +536,10 @@ public void testCanCache() throws Exception {
             .get();
         OpenSearchAssertions.assertAllSuccessful(r3);
         assertThat(r3.getHits().getTotalHits().value, equalTo(7L));
-        assertCacheState(client, "index", 0, 0);
+        assertCacheState(client, index, 0, 0);
 
         // If the request has an non-filter aggregation containing now we should not cache
-        final SearchResponse r5 = client.prepareSearch("index")
+        final SearchResponse r5 = client.prepareSearch(index)
             .setSearchType(SearchType.QUERY_THEN_FETCH)
             .setSize(0)
             .setRequestCache(true)
@@ -515,10 +548,10 @@ public void testCanCache() throws Exception {
             .get();
         OpenSearchAssertions.assertAllSuccessful(r5);
         assertThat(r5.getHits().getTotalHits().value, equalTo(7L));
-        assertCacheState(client, "index", 0, 0);
+        assertCacheState(client, index, 0, 0);
 
         // If size > 1 and cache flag is set on the request we should cache
-        final SearchResponse r6 = client.prepareSearch("index")
+        final SearchResponse r6 = client.prepareSearch(index)
             .setSearchType(SearchType.QUERY_THEN_FETCH)
             .setSize(1)
             .setRequestCache(true)
@@ -526,10 +559,10 @@ public void testCanCache() throws Exception {
             .get();
         OpenSearchAssertions.assertAllSuccessful(r6);
         assertThat(r6.getHits().getTotalHits().value, equalTo(7L));
-        assertCacheState(client, "index", 0, 2);
+        assertCacheState(client, index, 0, 2);
 
         // If the request has a filter aggregation containing now we should cache since it gets rewritten
-        final SearchResponse r4 = client.prepareSearch("index")
+        final SearchResponse r4 = client.prepareSearch(index)
             .setSearchType(SearchType.QUERY_THEN_FETCH)
             .setSize(0)
             .setRequestCache(true)
@@ -538,7 +571,7 @@ public void testCanCache() throws Exception {
             .get();
         OpenSearchAssertions.assertAllSuccessful(r4);
         assertThat(r4.getHits().getTotalHits().value, equalTo(7L));
-        assertCacheState(client, "index", 0, 4);
+        assertCacheState(client, index, 0, 4);
     }
 
     public void testCacheWithFilteredAlias() throws InterruptedException {
@@ -548,61 +581,63 @@ public void testCacheWithFilteredAlias() throws InterruptedException {
             .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
             .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
             .build();
+        String index = "index";
         assertAcked(
             client.admin()
                 .indices()
-                .prepareCreate("index")
+                .prepareCreate(index)
                 .setMapping("created_at", "type=date")
                 .setSettings(settings)
                 .addAlias(new Alias("last_week").filter(QueryBuilders.rangeQuery("created_at").gte("now-7d/d")))
                 .get()
         );
         ZonedDateTime now = ZonedDateTime.now(ZoneOffset.UTC);
-        client.prepareIndex("index").setId("1").setRouting("1").setSource("created_at", DateTimeFormatter.ISO_LOCAL_DATE.format(now)).get();
+        client.prepareIndex(index).setId("1").setRouting("1").setSource("created_at", DateTimeFormatter.ISO_LOCAL_DATE.format(now)).get();
         // Force merge the index to ensure there can be no background merges during the subsequent searches that would invalidate the cache
-        ForceMergeResponse forceMergeResponse = client.admin().indices().prepareForceMerge("index").setFlush(true).get();
+        ForceMergeResponse forceMergeResponse = client.admin().indices().prepareForceMerge(index).setFlush(true).get();
         OpenSearchAssertions.assertAllSuccessful(forceMergeResponse);
         refreshAndWaitForReplication();
 
-        indexRandomForConcurrentSearch("index");
+        indexRandomForConcurrentSearch(index);
 
-        assertCacheState(client, "index", 0, 0);
+        assertCacheState(client, index, 0, 0);
 
-        SearchResponse r1 = client.prepareSearch("index")
+        SearchResponse r1 = client.prepareSearch(index)
             .setSearchType(SearchType.QUERY_THEN_FETCH)
             .setSize(0)
             .setQuery(QueryBuilders.rangeQuery("created_at").gte("now-7d/d"))
             .get();
         OpenSearchAssertions.assertAllSuccessful(r1);
         assertThat(r1.getHits().getTotalHits().value, equalTo(1L));
-        assertCacheState(client, "index", 0, 1);
+        assertCacheState(client, index, 0, 1);
 
-        r1 = client.prepareSearch("index")
+        r1 = client.prepareSearch(index)
             .setSearchType(SearchType.QUERY_THEN_FETCH)
             .setSize(0)
             .setQuery(QueryBuilders.rangeQuery("created_at").gte("now-7d/d"))
             .get();
         OpenSearchAssertions.assertAllSuccessful(r1);
         assertThat(r1.getHits().getTotalHits().value, equalTo(1L));
-        assertCacheState(client, "index", 1, 1);
+        assertCacheState(client, index, 1, 1);
 
         r1 = client.prepareSearch("last_week").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0).get();
         OpenSearchAssertions.assertAllSuccessful(r1);
         assertThat(r1.getHits().getTotalHits().value, equalTo(1L));
-        assertCacheState(client, "index", 1, 2);
+        assertCacheState(client, index, 1, 2);
 
         r1 = client.prepareSearch("last_week").setSearchType(SearchType.QUERY_THEN_FETCH).setSize(0).get();
         OpenSearchAssertions.assertAllSuccessful(r1);
         assertThat(r1.getHits().getTotalHits().value, equalTo(1L));
-        assertCacheState(client, "index", 2, 2);
+        assertCacheState(client, index, 2, 2);
     }
 
     public void testProfileDisableCache() throws Exception {
         Client client = client();
+        String index = "index";
         assertAcked(
             client.admin()
                 .indices()
-                .prepareCreate("index")
+                .prepareCreate(index)
                 .setMapping("k", "type=keyword")
                 .setSettings(
                     Settings.builder()
@@ -612,14 +647,14 @@ public void testProfileDisableCache() throws Exception {
                 )
                 .get()
         );
-        indexRandom(true, client.prepareIndex("index").setSource("k", "hello"));
-        ensureSearchable("index");
+        indexRandom(true, client.prepareIndex(index).setSource("k", "hello"));
+        ensureSearchable(index);
 
         int expectedHits = 0;
         int expectedMisses = 0;
         for (int i = 0; i < 5; i++) {
             boolean profile = i % 2 == 0;
-            SearchResponse resp = client.prepareSearch("index")
+            SearchResponse resp = client.prepareSearch(index)
                 .setRequestCache(true)
                 .setProfile(profile)
                 .setQuery(QueryBuilders.termQuery("k", "hello"))
@@ -634,57 +669,709 @@ public void testProfileDisableCache() throws Exception {
                     expectedHits++;
                 }
             }
-            assertCacheState(client, "index", expectedHits, expectedMisses);
+            assertCacheState(client, index, expectedHits, expectedMisses);
         }
     }
 
     public void testCacheWithInvalidation() throws Exception {
         Client client = client();
+        String index = "index";
         assertAcked(
             client.admin()
                 .indices()
-                .prepareCreate("index")
+                .prepareCreate(index)
                 .setMapping("k", "type=keyword")
                 .setSettings(
                     Settings.builder()
                         .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
                         .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
                         .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                        .put("index.refresh_interval", -1)
                 )
                 .get()
         );
-        indexRandom(true, client.prepareIndex("index").setSource("k", "hello"));
-        ensureSearchable("index");
-        SearchResponse resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello")).get();
+        indexRandom(true, client.prepareIndex(index).setSource("k", "hello"));
+        ensureSearchable(index);
+        SearchResponse resp = client.prepareSearch(index).setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello")).get();
         assertSearchResponse(resp);
         OpenSearchAssertions.assertAllSuccessful(resp);
         assertThat(resp.getHits().getTotalHits().value, equalTo(1L));
 
-        assertCacheState(client, "index", 0, 1);
+        assertCacheState(client, index, 0, 1);
         // Index but don't refresh
-        indexRandom(false, client.prepareIndex("index").setSource("k", "hello2"));
-        resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello")).get();
+        indexRandom(false, client.prepareIndex(index).setSource("k", "hello2"));
+        resp = client.prepareSearch(index).setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello")).get();
         assertSearchResponse(resp);
         // Should expect hit as here as refresh didn't happen
-        assertCacheState(client, "index", 1, 1);
+        assertCacheState(client, index, 1, 1);
 
         // Explicit refresh would invalidate cache
         refreshAndWaitForReplication();
         // Hit same query again
-        resp = client.prepareSearch("index").setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello")).get();
+        resp = client.prepareSearch(index).setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello")).get();
         assertSearchResponse(resp);
         // Should expect miss as key has changed due to change in IndexReader.CacheKey (due to refresh)
-        assertCacheState(client, "index", 1, 2);
+        assertCacheState(client, index, 1, 2);
+    }
+
+    // calling cache clear api, when staleness threshold is lower than staleness, it should clean the stale keys from cache
+    public void testCacheClearAPIRemovesStaleKeysWhenStalenessThresholdIsLow() throws Exception {
+        String node = internalCluster().startNode(
+            Settings.builder()
+                .put(IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_STALENESS_THRESHOLD_SETTING_KEY, 0.10)
+                .put(
+                    IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_INTERVAL_SETTING_KEY,
+                    // setting intentionally high to avoid cache cleaner interfering
+                    TimeValue.timeValueMillis(300)
+                )
+        );
+        Client client = client(node);
+        String index1 = "index1";
+        String index2 = "index2";
+        setupIndex(client, index1);
+        setupIndex(client, index2);
+
+        // create first cache entry in index1
+        createCacheEntry(client, index1, "hello");
+        assertCacheState(client, index1, 0, 1);
+        long memorySizeForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
+        assertTrue(memorySizeForIndex1 > 0);
+
+        // create second cache entry in index1
+        createCacheEntry(client, index1, "there");
+        assertCacheState(client, index1, 0, 2);
+        long finalMemorySizeForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
+        assertTrue(finalMemorySizeForIndex1 > memorySizeForIndex1);
+
+        // create first cache entry in index2
+        createCacheEntry(client, index2, "hello");
+        assertCacheState(client, index2, 0, 1);
+        assertTrue(getRequestCacheStats(client, index2).getMemorySizeInBytes() > 0);
+
+        ClearIndicesCacheRequest clearIndicesCacheRequest = new ClearIndicesCacheRequest(index2);
+        client.admin().indices().clearCache(clearIndicesCacheRequest).actionGet();
+
+        // cache cleaner should have cleaned up the stale key from index 2
+        assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes());
+        // cache cleaner should NOT have cleaned from index 1
+        assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes());
+    }
+
+    // when staleness threshold is lower than staleness, it should clean the stale keys from cache
+    public void testStaleKeysCleanupWithLowThreshold() throws Exception {
+        int cacheCleanIntervalInMillis = 1;
+        String node = internalCluster().startNode(
+            Settings.builder()
+                .put(IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_STALENESS_THRESHOLD_SETTING_KEY, 0.10)
+                .put(
+                    IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_INTERVAL_SETTING_KEY,
+                    TimeValue.timeValueMillis(cacheCleanIntervalInMillis)
+                )
+        );
+        Client client = client(node);
+        String index1 = "index1";
+        String index2 = "index2";
+        setupIndex(client, index1);
+        setupIndex(client, index2);
+
+        // create first cache entry in index1
+        createCacheEntry(client, index1, "hello");
+        assertCacheState(client, index1, 0, 1);
+        long memorySizeForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
+        assertTrue(memorySizeForIndex1 > 0);
+
+        // create second cache entry in index1
+        createCacheEntry(client, index1, "there");
+        assertCacheState(client, index1, 0, 2);
+        long finalMemorySizeForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
+        assertTrue(finalMemorySizeForIndex1 > memorySizeForIndex1);
+
+        // create first cache entry in index2
+        createCacheEntry(client, index2, "hello");
+        assertCacheState(client, index2, 0, 1);
+        assertTrue(getRequestCacheStats(client, index2).getMemorySizeInBytes() > 0);
+
+        // force refresh so that it creates 1 stale key
+        flushAndRefresh(index2);
+        // sleep until cache cleaner would have cleaned up the stale key from index 2
+        assertBusy(() -> {
+            // cache cleaner should have cleaned up the stale key from index 2
+            assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes());
+            // cache cleaner should NOT have cleaned from index 1
+            assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes());
+        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
+        // sleep until cache cleaner would have cleaned up the stale key from index 2
+    }
+
+    // when staleness threshold is equal to staleness, it should clean the stale keys from cache
+    public void testCacheCleanupOnEqualStalenessAndThreshold() throws Exception {
+        int cacheCleanIntervalInMillis = 1;
+        String node = internalCluster().startNode(
+            Settings.builder()
+                .put(IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_STALENESS_THRESHOLD_SETTING_KEY, 0.33)
+                .put(
+                    IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_INTERVAL_SETTING_KEY,
+                    TimeValue.timeValueMillis(cacheCleanIntervalInMillis)
+                )
+        );
+        Client client = client(node);
+        String index1 = "index1";
+        String index2 = "index2";
+        setupIndex(client, index1);
+        setupIndex(client, index2);
+
+        // create first cache entry in index1
+        createCacheEntry(client, index1, "hello");
+        assertCacheState(client, index1, 0, 1);
+        long memorySizeForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
+        assertTrue(memorySizeForIndex1 > 0);
+
+        // create second cache entry in index1
+        createCacheEntry(client, index1, "there");
+        assertCacheState(client, index1, 0, 2);
+        long finalMemorySizeForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
+        assertTrue(finalMemorySizeForIndex1 > memorySizeForIndex1);
+
+        // create first cache entry in index2
+        createCacheEntry(client, index2, "hello");
+        assertCacheState(client, index2, 0, 1);
+        assertTrue(getRequestCacheStats(client, index2).getMemorySizeInBytes() > 0);
+
+        // force refresh so that it creates 1 stale key
+        flushAndRefresh(index2);
+        // sleep until cache cleaner would have cleaned up the stale key from index 2
+        assertBusy(() -> {
+            // cache cleaner should have cleaned up the stale key from index 2
+            assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes());
+            // cache cleaner should NOT have cleaned from index 1
+            assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes());
+        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
+    }
+
+    // when staleness threshold is higher than staleness, it should NOT clean the cache
+    public void testCacheCleanupSkipsWithHighStalenessThreshold() throws Exception {
+        int cacheCleanIntervalInMillis = 1;
+        String node = internalCluster().startNode(
+            Settings.builder()
+                .put(IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_STALENESS_THRESHOLD_SETTING_KEY, 0.90)
+                .put(
+                    IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_INTERVAL_SETTING_KEY,
+                    TimeValue.timeValueMillis(cacheCleanIntervalInMillis)
+                )
+        );
+        Client client = client(node);
+        String index1 = "index1";
+        String index2 = "index2";
+        setupIndex(client, index1);
+        setupIndex(client, index2);
+
+        // create first cache entry in index1
+        createCacheEntry(client, index1, "hello");
+        assertCacheState(client, index1, 0, 1);
+        long memorySizeForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
+        assertTrue(memorySizeForIndex1 > 0);
+
+        // create second cache entry in index1
+        createCacheEntry(client, index1, "there");
+        assertCacheState(client, index1, 0, 2);
+        long finalMemorySizeForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
+        assertTrue(finalMemorySizeForIndex1 > memorySizeForIndex1);
+
+        // create first cache entry in index2
+        createCacheEntry(client, index2, "hello");
+        assertCacheState(client, index2, 0, 1);
+        assertTrue(getRequestCacheStats(client, index2).getMemorySizeInBytes() > 0);
+
+        // force refresh so that it creates 1 stale key
+        flushAndRefresh(index2);
+        // sleep until cache cleaner would have cleaned up the stale key from index 2
+        assertBusy(() -> {
+            // cache cleaner should NOT have cleaned up the stale key from index 2
+            assertTrue(getRequestCacheStats(client, index2).getMemorySizeInBytes() > 0);
+            // cache cleaner should NOT have cleaned from index 1
+            assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes());
+        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
+    }
+
+    // when staleness threshold is explicitly set to 0, cache cleaner regularly cleans up stale keys.
+    public void testCacheCleanupOnZeroStalenessThreshold() throws Exception {
+        int cacheCleanIntervalInMillis = 50;
+        String node = internalCluster().startNode(
+            Settings.builder()
+                .put(IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_STALENESS_THRESHOLD_SETTING_KEY, 0)
+                .put(
+                    IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_INTERVAL_SETTING_KEY,
+                    TimeValue.timeValueMillis(cacheCleanIntervalInMillis)
+                )
+        );
+        Client client = client(node);
+        String index1 = "index1";
+        String index2 = "index2";
+        setupIndex(client, index1);
+        setupIndex(client, index2);
+
+        // create 10 index1 cache entries
+        for (int i = 1; i <= 10; i++) {
+            long cacheSizeBefore = getRequestCacheStats(client, index1).getMemorySizeInBytes();
+            createCacheEntry(client, index1, "hello" + i);
+            assertCacheState(client, index1, 0, i);
+            long cacheSizeAfter = getRequestCacheStats(client, index1).getMemorySizeInBytes();
+            assertTrue(cacheSizeAfter > cacheSizeBefore);
+        }
+
+        long finalMemorySizeForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
+
+        // create first cache entry in index2
+        createCacheEntry(client, index2, "hello");
+        assertCacheState(client, index2, 0, 1);
+        assertTrue(getRequestCacheStats(client, index2).getMemorySizeInBytes() > 0);
+
+        // force refresh so that it creates 1 stale key
+        flushAndRefresh(index2);
+        // sleep until cache cleaner would have cleaned up the stale key from index 2
+        assertBusy(() -> {
+            // cache cleaner should have cleaned up the stale key from index 2
+            assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes());
+            // cache cleaner should NOT have cleaned from index 1
+            assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes());
+        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
+    }
+
+    // when staleness threshold is not explicitly set, cache cleaner regularly cleans up stale keys
+    public void testStaleKeysRemovalWithoutExplicitThreshold() throws Exception {
+        int cacheCleanIntervalInMillis = 1;
+        String node = internalCluster().startNode(
+            Settings.builder()
+                .put(
+                    IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_INTERVAL_SETTING_KEY,
+                    TimeValue.timeValueMillis(cacheCleanIntervalInMillis)
+                )
+        );
+        String index1 = "index1";
+        String index2 = "index2";
+        Client client = client(node);
+        setupIndex(client, index1);
+        setupIndex(client, index2);
+
+        // create first cache entry in index1
+        createCacheEntry(client, index1, "hello");
+        assertCacheState(client, index1, 0, 1);
+        long memorySizeForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
+        assertTrue(memorySizeForIndex1 > 0);
+
+        // create second cache entry in index1
+        createCacheEntry(client, index1, "there");
+        assertCacheState(client, index1, 0, 2);
+        long finalMemorySizeForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
+        assertTrue(finalMemorySizeForIndex1 > memorySizeForIndex1);
+
+        // create first cache entry in index2
+        createCacheEntry(client, index2, "hello");
+        assertCacheState(client, index2, 0, 1);
+        assertTrue(getRequestCacheStats(client, index2).getMemorySizeInBytes() > 0);
+
+        // force refresh so that it creates 1 stale key
+        flushAndRefresh(index2);
+        // sleep until cache cleaner would have cleaned up the stale key from index 2
+        assertBusy(() -> {
+            // cache cleaner should have cleaned up the stale key from index 2
+            assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes());
+            // cache cleaner should NOT have cleaned from index 1
+            assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes());
+        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
+    }
+
+    // when cache cleaner interval setting is not set, cache cleaner is configured appropriately with the fall-back setting
+    public void testCacheCleanupWithDefaultSettings() throws Exception {
+        int cacheCleanIntervalInMillis = 1;
+        String node = internalCluster().startNode(
+            Settings.builder().put(INDICES_CACHE_CLEANUP_INTERVAL_SETTING_KEY, TimeValue.timeValueMillis(cacheCleanIntervalInMillis))
+        );
+        Client client = client(node);
+        String index1 = "index1";
+        String index2 = "index2";
+        setupIndex(client, index1);
+        setupIndex(client, index2);
+
+        // create first cache entry in index1
+        createCacheEntry(client, index1, "hello");
+        assertCacheState(client, index1, 0, 1);
+        long memorySizeForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
+        assertTrue(memorySizeForIndex1 > 0);
+
+        // create second cache entry in index1
+        createCacheEntry(client, index1, "there");
+        assertCacheState(client, index1, 0, 2);
+        long finalMemorySizeForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
+        assertTrue(finalMemorySizeForIndex1 > memorySizeForIndex1);
+
+        // create first cache entry in index2
+        createCacheEntry(client, index2, "hello");
+        assertCacheState(client, index2, 0, 1);
+        assertTrue(getRequestCacheStats(client, index2).getMemorySizeInBytes() > 0);
+
+        // force refresh so that it creates 1 stale key
+        flushAndRefresh(index2);
+        // sleep until cache cleaner would have cleaned up the stale key from index 2
+        assertBusy(() -> {
+            // cache cleaner should have cleaned up the stale key from index 2
+            assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes());
+            // cache cleaner should NOT have cleaned from index 1
+            assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes());
+        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
+    }
+
+    // staleness threshold updates flows through to the cache cleaner
+    public void testDynamicStalenessThresholdUpdate() throws Exception {
+        int cacheCleanIntervalInMillis = 1;
+        String node = internalCluster().startNode(
+            Settings.builder()
+                .put(IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_STALENESS_THRESHOLD_SETTING_KEY, 0.90)
+                .put(
+                    IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_INTERVAL_SETTING_KEY,
+                    TimeValue.timeValueMillis(cacheCleanIntervalInMillis)
+                )
+        );
+        Client client = client(node);
+        String index1 = "index1";
+        String index2 = "index2";
+        setupIndex(client, index1);
+        setupIndex(client, index2);
+
+        // create first cache entry in index1
+        createCacheEntry(client, index1, "hello");
+        assertCacheState(client, index1, 0, 1);
+        long memorySizeForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
+        assertTrue(memorySizeForIndex1 > 0);
+
+        // create second cache entry in index1
+        createCacheEntry(client, index1, "there");
+        assertCacheState(client, index1, 0, 2);
+        assertTrue(getRequestCacheStats(client, index1).getMemorySizeInBytes() > memorySizeForIndex1);
+
+        // create first cache entry in index2
+        createCacheEntry(client, index2, "hello");
+        assertCacheState(client, index2, 0, 1);
+        long finalMemorySizeForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
+        assertTrue(finalMemorySizeForIndex1 > 0);
+
+        // force refresh so that it creates 1 stale key
+        flushAndRefresh(index2);
+        assertBusy(() -> {
+            // cache cleaner should NOT have cleaned up the stale key from index 2
+            assertTrue(getRequestCacheStats(client, index2).getMemorySizeInBytes() > 0);
+        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
+
+        // Update indices.requests.cache.cleanup.staleness_threshold to "10%"
+        ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
+        updateSettingsRequest.persistentSettings(Settings.builder().put(INDICES_REQUEST_CACHE_STALENESS_THRESHOLD_SETTING.getKey(), 0.10));
+        assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
+
+        assertBusy(() -> {
+            // cache cleaner should have cleaned up the stale key from index 2
+            assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes());
+            // cache cleaner should NOT have cleaned from index 1
+            assertEquals(finalMemorySizeForIndex1, getRequestCacheStats(client, index1).getMemorySizeInBytes());
+        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
+    }
+
+    // staleness threshold dynamic updates should throw exceptions on invalid input
+    public void testInvalidStalenessThresholdUpdateThrowsException() throws Exception {
+        int cacheCleanIntervalInMillis = 1;
+        String node = internalCluster().startNode(
+            Settings.builder()
+                .put(IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_STALENESS_THRESHOLD_SETTING_KEY, 0.90)
+                .put(
+                    IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_INTERVAL_SETTING_KEY,
+                    TimeValue.timeValueMillis(cacheCleanIntervalInMillis)
+                )
+        );
+        Client client = client(node);
+        String index1 = "index1";
+        setupIndex(client, index1);
+
+        // create first cache entry in index1
+        createCacheEntry(client, index1, "hello");
+        assertCacheState(client, index1, 0, 1);
+        assertTrue(getRequestCacheStats(client, index1).getMemorySizeInBytes() > 0);
+
+        // Update indices.requests.cache.cleanup.staleness_threshold to "10%" with illegal argument
+        assertThrows("Ratio should be in [0-1.0]", IllegalArgumentException.class, () -> {
+            ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
+            updateSettingsRequest.persistentSettings(
+                Settings.builder().put(IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_STALENESS_THRESHOLD_SETTING_KEY, 10)
+            );
+            client().admin().cluster().updateSettings(updateSettingsRequest).actionGet();
+        });
+
+        // everything else should continue to work fine later on.
+        // force refresh so that it creates 1 stale key
+        flushAndRefresh(index1);
+        // sleep until cache cleaner would have cleaned up the stale key from index 2
+        assertBusy(() -> {
+            // cache cleaner should NOT have cleaned from index 1
+            assertEquals(0, getRequestCacheStats(client, index1).getMemorySizeInBytes());
+        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
+    }
+
+    // closing the Index after caching will clean up from Indices Request Cache
+    public void testCacheClearanceAfterIndexClosure() throws Exception {
+        int cacheCleanIntervalInMillis = 100;
+        String node = internalCluster().startNode(
+            Settings.builder()
+                .put(IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_STALENESS_THRESHOLD_SETTING_KEY, 0.10)
+                .put(
+                    IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_INTERVAL_SETTING_KEY,
+                    TimeValue.timeValueMillis(cacheCleanIntervalInMillis)
+                )
+        );
+        Client client = client(node);
+        String index = "index";
+        setupIndex(client, index);
+
+        // assert there are no entries in the cache for index
+        assertEquals(0, getRequestCacheStats(client, index).getMemorySizeInBytes());
+        // assert there are no entries in the cache from other indices in the node
+        assertEquals(0, getNodeCacheStats(client).getMemorySizeInBytes());
+        // create first cache entry in index
+        createCacheEntry(client, index, "hello");
+        assertCacheState(client, index, 0, 1);
+        assertTrue(getRequestCacheStats(client, index).getMemorySizeInBytes() > 0);
+        assertTrue(getNodeCacheStats(client).getMemorySizeInBytes() > 0);
+
+        // close index
+        assertAcked(client.admin().indices().prepareClose(index));
+        // request cache stats cannot be access since Index should be closed
+        try {
+            getRequestCacheStats(client, index);
+        } catch (Exception e) {
+            assert (e instanceof IndexClosedException);
+        }
+        // sleep until cache cleaner would have cleaned up the stale key from index
+        assertBusy(() -> {
+            // cache cleaner should have cleaned up the stale keys from index
+            assertEquals(0, getNodeCacheStats(client).getMemorySizeInBytes());
+        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
+    }
+
+    // deleting the Index after caching will clean up from Indices Request Cache
+    public void testCacheCleanupAfterIndexDeletion() throws Exception {
+        int cacheCleanIntervalInMillis = 100;
+        String node = internalCluster().startNode(
+            Settings.builder()
+                .put(IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_STALENESS_THRESHOLD_SETTING_KEY, 0.10)
+                .put(
+                    IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_INTERVAL_SETTING_KEY,
+                    TimeValue.timeValueMillis(cacheCleanIntervalInMillis)
+                )
+        );
+        Client client = client(node);
+        String index = "index";
+        setupIndex(client, index);
+
+        // assert there are no entries in the cache for index
+        assertEquals(0, getRequestCacheStats(client, index).getMemorySizeInBytes());
+        // assert there are no entries in the cache from other indices in the node
+        assertEquals(0, getNodeCacheStats(client).getMemorySizeInBytes());
+        // create first cache entry in index
+        createCacheEntry(client, index, "hello");
+        assertCacheState(client, index, 0, 1);
+        assertTrue(getRequestCacheStats(client, index).getMemorySizeInBytes() > 0);
+        assertTrue(getNodeCacheStats(client).getMemorySizeInBytes() > 0);
+
+        // delete index
+        assertAcked(client.admin().indices().prepareDelete(index));
+        // request cache stats cannot be access since Index should be deleted
+        try {
+            getRequestCacheStats(client, index);
+        } catch (Exception e) {
+            assert (e instanceof IndexNotFoundException);
+        }
+
+        // sleep until cache cleaner would have cleaned up the stale key from index
+        assertBusy(() -> {
+            // cache cleaner should have cleaned up the stale keys from index
+            assertEquals(0, getNodeCacheStats(client).getMemorySizeInBytes());
+        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
+    }
+
+    // when staleness threshold is lower than staleness, it should clean the cache from all indices having stale keys
+    public void testStaleKeysCleanupWithMultipleIndices() throws Exception {
+        int cacheCleanIntervalInMillis = 10;
+        String node = internalCluster().startNode(
+            Settings.builder()
+                .put(IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_STALENESS_THRESHOLD_SETTING_KEY, 0.10)
+                .put(
+                    IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_INTERVAL_SETTING_KEY,
+                    TimeValue.timeValueMillis(cacheCleanIntervalInMillis)
+                )
+        );
+        Client client = client(node);
+        String index1 = "index1";
+        String index2 = "index2";
+        setupIndex(client, index1);
+        setupIndex(client, index2);
+
+        // assert cache is empty for index1
+        assertEquals(0, getRequestCacheStats(client, index1).getMemorySizeInBytes());
+        // create first cache entry in index1
+        createCacheEntry(client, index1, "hello");
+        assertCacheState(client, index1, 0, 1);
+        long memorySizeForIndex1With1Entries = getRequestCacheStats(client, index1).getMemorySizeInBytes();
+        assertTrue(memorySizeForIndex1With1Entries > 0);
+
+        // create second cache entry in index1
+        createCacheEntry(client, index1, "there");
+        assertCacheState(client, index1, 0, 2);
+        long memorySizeForIndex1With2Entries = getRequestCacheStats(client, index1).getMemorySizeInBytes();
+        assertTrue(memorySizeForIndex1With2Entries > memorySizeForIndex1With1Entries);
+
+        // assert cache is empty for index2
+        assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes());
+        // create first cache entry in index2
+        createCacheEntry(client, index2, "hello");
+        assertCacheState(client, index2, 0, 1);
+        assertTrue(getRequestCacheStats(client, index2).getMemorySizeInBytes() > 0);
+
+        // force refresh both index1 and index2
+        flushAndRefresh(index1, index2);
+        // create another cache entry in index 1 same as memorySizeForIndex1With1Entries, this should not be cleaned up.
+        createCacheEntry(client, index1, "hello");
+        // sleep until cache cleaner would have cleaned up the stale key from index2
+        assertBusy(() -> {
+            // cache cleaner should have cleaned up the stale key from index2 and hence cache should be empty
+            assertEquals(0, getRequestCacheStats(client, index2).getMemorySizeInBytes());
+            // cache cleaner should have only cleaned up the stale entities for index1
+            long currentMemorySizeInBytesForIndex1 = getRequestCacheStats(client, index1).getMemorySizeInBytes();
+            // assert the memory size of index1 to only contain 1 entry added after flushAndRefresh
+            assertEquals(memorySizeForIndex1With1Entries, currentMemorySizeInBytesForIndex1);
+            // cache for index1 should not be empty since there was an item cached after flushAndRefresh
+            assertTrue(currentMemorySizeInBytesForIndex1 > 0);
+        }, cacheCleanIntervalInMillis * 2, TimeUnit.MILLISECONDS);
+    }
+
+    public void testDeleteAndCreateSameIndexShardOnSameNode() throws Exception {
+        String node_1 = internalCluster().startNode(Settings.builder().build());
+        Client client = client(node_1);
+
+        logger.info("Starting a node in the cluster");
+
+        assertThat(cluster().size(), equalTo(1));
+        ClusterHealthResponse healthResponse = client().admin().cluster().prepareHealth().setWaitForNodes("1").execute().actionGet();
+        assertThat(healthResponse.isTimedOut(), equalTo(false));
+
+        String indexName = "test";
+
+        logger.info("Creating an index: {} with 2 shards", indexName);
+        createIndex(
+            indexName,
+            Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build()
+        );
+
+        ensureGreen(indexName);
+
+        logger.info("Writing few docs and searching those which will cache items in RequestCache");
+        indexRandom(true, client.prepareIndex(indexName).setSource("k", "hello"));
+        indexRandom(true, client.prepareIndex(indexName).setSource("y", "hello again"));
+        SearchResponse resp = client.prepareSearch(indexName).setRequestCache(true).setQuery(QueryBuilders.termQuery("k", "hello")).get();
+        assertSearchResponse(resp);
+        resp = client.prepareSearch(indexName).setRequestCache(true).setQuery(QueryBuilders.termQuery("y", "hello")).get();
+
+        RequestCacheStats stats = getNodeCacheStats(client);
+        assertTrue(stats.getMemorySizeInBytes() > 0);
+
+        logger.info("Disabling allocation");
+        Settings newSettings = Settings.builder()
+            .put(CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING.getKey(), EnableAllocationDecider.Allocation.NONE.name())
+            .build();
+        client().admin().cluster().prepareUpdateSettings().setTransientSettings(newSettings).execute().actionGet();
+
+        logger.info("Starting a second node");
+        String node_2 = internalCluster().startDataOnlyNode(Settings.builder().build());
+        assertThat(cluster().size(), equalTo(2));
+        healthResponse = client().admin().cluster().prepareHealth().setWaitForNodes("2").execute().actionGet();
+        assertThat(healthResponse.isTimedOut(), equalTo(false));
+
+        logger.info("Moving the shard:{} from node:{} to node:{}", indexName + "#0", node_1, node_2);
+        MoveAllocationCommand cmd = new MoveAllocationCommand(indexName, 0, node_1, node_2);
+        internalCluster().client().admin().cluster().prepareReroute().add(cmd).get();
+        ClusterHealthResponse clusterHealth = client().admin()
+            .cluster()
+            .prepareHealth()
+            .setWaitForNoRelocatingShards(true)
+            .setWaitForNoInitializingShards(true)
+            .get();
+        assertThat(clusterHealth.isTimedOut(), equalTo(false));
+
+        ClusterState state = client().admin().cluster().prepareState().get().getState();
+        final Index index = state.metadata().index(indexName).getIndex();
+
+        assertBusy(() -> {
+            assertThat(Files.exists(shardDirectory(node_1, index, 0)), equalTo(false));
+            assertThat(Files.exists(shardDirectory(node_2, index, 0)), equalTo(true));
+        });
+
+        logger.info("Moving the shard: {} again from node:{} to node:{}", indexName + "#0", node_2, node_1);
+        cmd = new MoveAllocationCommand(indexName, 0, node_2, node_1);
+        internalCluster().client().admin().cluster().prepareReroute().add(cmd).get();
+        clusterHealth = client().admin()
+            .cluster()
+            .prepareHealth()
+            .setWaitForNoRelocatingShards(true)
+            .setWaitForNoInitializingShards(true)
+            .get();
+        assertThat(clusterHealth.isTimedOut(), equalTo(false));
+        assertThat(Files.exists(shardDirectory(node_1, index, 0)), equalTo(true));
+
+        assertBusy(() -> {
+            assertThat(Files.exists(shardDirectory(node_1, index, 0)), equalTo(true));
+            assertThat(Files.exists(shardDirectory(node_2, index, 0)), equalTo(false));
+        });
+
+        logger.info("Clearing the cache for index:{}. And verify the request stats doesn't go negative", indexName);
+        ClearIndicesCacheRequest clearIndicesCacheRequest = new ClearIndicesCacheRequest(indexName);
+        client.admin().indices().clearCache(clearIndicesCacheRequest).actionGet();
+
+        stats = getNodeCacheStats(client(node_1));
+        assertTrue(stats.getMemorySizeInBytes() == 0);
+        stats = getNodeCacheStats(client(node_2));
+        assertTrue(stats.getMemorySizeInBytes() == 0);
+    }
+
+    private Path shardDirectory(String server, Index index, int shard) {
+        NodeEnvironment env = internalCluster().getInstance(NodeEnvironment.class, server);
+        final Path[] paths = env.availableShardPaths(new ShardId(index, shard));
+        assert paths.length == 1;
+        return paths[0];
+    }
+
+    private void setupIndex(Client client, String index) throws Exception {
+        assertAcked(
+            client.admin()
+                .indices()
+                .prepareCreate(index)
+                .setMapping("k", "type=keyword")
+                .setSettings(
+                    Settings.builder()
+                        .put(IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), true)
+                        .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                        .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                )
+                .get()
+        );
+        indexRandom(true, client.prepareIndex(index).setSource("k", "hello"));
+        indexRandom(true, client.prepareIndex(index).setSource("k", "there"));
+        ensureSearchable(index);
+    }
+
+    private void createCacheEntry(Client client, String index, String value) {
+        SearchResponse resp = client.prepareSearch(index).setRequestCache(true).setQuery(QueryBuilders.termQuery("k", value)).get();
+        assertSearchResponse(resp);
+        OpenSearchAssertions.assertAllSuccessful(resp);
     }
 
     private static void assertCacheState(Client client, String index, long expectedHits, long expectedMisses) {
-        RequestCacheStats requestCacheStats = client.admin()
-            .indices()
-            .prepareStats(index)
-            .setRequestCache(true)
-            .get()
-            .getTotal()
-            .getRequestCache();
+        RequestCacheStats requestCacheStats = getRequestCacheStats(client, index);
         // Check the hit count and miss count together so if they are not
         // correct we can see both values
         assertEquals(
@@ -694,4 +1381,17 @@ private static void assertCacheState(Client client, String index, long expectedH
 
     }
 
+    private static RequestCacheStats getRequestCacheStats(Client client, String index) {
+        return client.admin().indices().prepareStats(index).setRequestCache(true).get().getTotal().getRequestCache();
+    }
+
+    private static RequestCacheStats getNodeCacheStats(Client client) {
+        NodesStatsResponse stats = client.admin().cluster().prepareNodesStats().execute().actionGet();
+        for (NodeStats stat : stats.getNodes()) {
+            if (stat.getNode().isDataNode()) {
+                return stat.getIndices().getRequestCache();
+            }
+        }
+        return null;
+    }
 }
diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/recovery/ReplicaToPrimaryPromotionIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/recovery/ReplicaToPrimaryPromotionIT.java
index 3df4ecff5250c..a2543f0592145 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/recovery/ReplicaToPrimaryPromotionIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/recovery/ReplicaToPrimaryPromotionIT.java
@@ -56,6 +56,11 @@ protected int numberOfReplicas() {
         return 1;
     }
 
+    @Override
+    public boolean useRandomReplicationStrategy() {
+        return true;
+    }
+
     public void testPromoteReplicaToPrimary() throws Exception {
         final String indexName = randomAlphaOfLength(5).toLowerCase(Locale.ROOT);
         createIndex(indexName);
@@ -65,7 +70,7 @@ public void testPromoteReplicaToPrimary() throws Exception {
             try (BackgroundIndexer indexer = new BackgroundIndexer(indexName, "_doc", client(), numOfDocs)) {
                 waitForDocs(numOfDocs, indexer);
             }
-            refresh(indexName);
+            refreshAndWaitForReplication(indexName);
         }
 
         assertHitCount(client().prepareSearch(indexName).setSize(0).get(), numOfDocs);
diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationAllocationIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationAllocationIT.java
index 30edea6551067..669e24f9fb555 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationAllocationIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationAllocationIT.java
@@ -31,6 +31,9 @@
 import java.util.stream.Collectors;
 
 import static org.opensearch.cluster.routing.ShardRoutingState.STARTED;
+import static org.opensearch.cluster.routing.allocation.allocator.BalancedShardsAllocator.PREFER_PRIMARY_SHARD_BALANCE;
+import static org.opensearch.cluster.routing.allocation.allocator.BalancedShardsAllocator.PREFER_PRIMARY_SHARD_REBALANCE;
+import static org.opensearch.cluster.routing.allocation.allocator.BalancedShardsAllocator.PRIMARY_SHARD_REBALANCE_BUFFER;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
 
 @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
@@ -58,6 +61,20 @@ public void enablePreferPrimaryBalance() {
         );
     }
 
+    public void setAllocationRelocationStrategy(boolean preferPrimaryBalance, boolean preferPrimaryRebalance, float buffer) {
+        assertAcked(
+            client().admin()
+                .cluster()
+                .prepareUpdateSettings()
+                .setPersistentSettings(
+                    Settings.builder()
+                        .put(PREFER_PRIMARY_SHARD_BALANCE.getKey(), preferPrimaryBalance)
+                        .put(PREFER_PRIMARY_SHARD_REBALANCE.getKey(), preferPrimaryRebalance)
+                        .put(PRIMARY_SHARD_REBALANCE_BUFFER.getKey(), buffer)
+                )
+        );
+    }
+
     /**
      * This test verifies that the overall primary balance is attained during allocation. This test verifies primary
      * balance per index and across all indices is maintained.
@@ -87,7 +104,7 @@ public void testGlobalPrimaryAllocation() throws Exception {
         state = client().admin().cluster().prepareState().execute().actionGet().getState();
         logger.info(ShardAllocations.printShardDistribution(state));
         verifyPerIndexPrimaryBalance();
-        verifyPrimaryBalance();
+        verifyPrimaryBalance(0.0f);
     }
 
     /**
@@ -224,6 +241,70 @@ public void testAllocationWithDisruption() throws Exception {
         verifyPerIndexPrimaryBalance();
     }
 
+    /**
+     * Similar to testSingleIndexShardAllocation test but creates multiple indices, multiple nodes adding in and getting
+     * removed. The test asserts post each such event that primary shard distribution is balanced for each index as well as across the nodes
+     * when the PREFER_PRIMARY_SHARD_REBALANCE is set to true
+     */
+    public void testAllocationAndRebalanceWithDisruption() throws Exception {
+        internalCluster().startClusterManagerOnlyNode();
+        final int maxReplicaCount = 2;
+        final int maxShardCount = 2;
+        // Create higher number of nodes than number of shards to reduce chances of SameShardAllocationDecider kicking-in
+        // and preventing primary relocations
+        final int nodeCount = randomIntBetween(5, 10);
+        final int numberOfIndices = randomIntBetween(1, 10);
+        final float buffer = randomIntBetween(1, 4) * 0.10f;
+
+        logger.info("--> Creating {} nodes", nodeCount);
+        final List<String> nodeNames = new ArrayList<>();
+        for (int i = 0; i < nodeCount; i++) {
+            nodeNames.add(internalCluster().startNode());
+        }
+        setAllocationRelocationStrategy(true, true, buffer);
+
+        int shardCount, replicaCount;
+        ClusterState state;
+        for (int i = 0; i < numberOfIndices; i++) {
+            shardCount = randomIntBetween(1, maxShardCount);
+            replicaCount = randomIntBetween(1, maxReplicaCount);
+            logger.info("--> Creating index test{} with primary {} and replica {}", i, shardCount, replicaCount);
+            createIndex("test" + i, shardCount, replicaCount, i % 2 == 0);
+            ensureGreen(TimeValue.timeValueSeconds(60));
+            if (logger.isTraceEnabled()) {
+                state = client().admin().cluster().prepareState().execute().actionGet().getState();
+                logger.info(ShardAllocations.printShardDistribution(state));
+            }
+        }
+        state = client().admin().cluster().prepareState().execute().actionGet().getState();
+        logger.info(ShardAllocations.printShardDistribution(state));
+        verifyPerIndexPrimaryBalance();
+        verifyPrimaryBalance(buffer);
+
+        final int additionalNodeCount = randomIntBetween(1, 5);
+        logger.info("--> Adding {} nodes", additionalNodeCount);
+
+        internalCluster().startNodes(additionalNodeCount);
+        ensureGreen(TimeValue.timeValueSeconds(60));
+        state = client().admin().cluster().prepareState().execute().actionGet().getState();
+        logger.info(ShardAllocations.printShardDistribution(state));
+        verifyPerIndexPrimaryBalance();
+        verifyPrimaryBalance(buffer);
+
+        int nodeCountToStop = additionalNodeCount;
+        while (nodeCountToStop > 0) {
+            internalCluster().stopRandomDataNode();
+            // give replica a chance to promote as primary before terminating node containing the replica
+            ensureGreen(TimeValue.timeValueSeconds(60));
+            nodeCountToStop--;
+        }
+        state = client().admin().cluster().prepareState().execute().actionGet().getState();
+        logger.info("--> Cluster state post nodes stop {}", state);
+        logger.info(ShardAllocations.printShardDistribution(state));
+        verifyPerIndexPrimaryBalance();
+        verifyPrimaryBalance(buffer);
+    }
+
     /**
      * Utility method which ensures cluster has balanced primary shard distribution across a single index.
      * @throws Exception exception
@@ -263,7 +344,7 @@ private void verifyPerIndexPrimaryBalance() throws Exception {
         }, 60, TimeUnit.SECONDS);
     }
 
-    private void verifyPrimaryBalance() throws Exception {
+    private void verifyPrimaryBalance(float buffer) throws Exception {
         assertBusy(() -> {
             final ClusterState currentState = client().admin().cluster().prepareState().execute().actionGet().getState();
             RoutingNodes nodes = currentState.getRoutingNodes();
@@ -278,7 +359,7 @@ private void verifyPrimaryBalance() throws Exception {
                     .filter(ShardRouting::primary)
                     .collect(Collectors.toList())
                     .size();
-                assertTrue(primaryCount <= avgPrimaryShardsPerNode);
+                assertTrue(primaryCount <= (avgPrimaryShardsPerNode * (1 + buffer)));
             }
         }, 60, TimeUnit.SECONDS);
     }
diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationSuiteIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationSuiteIT.java
index 8c045c1560dd3..27b65432e0bac 100644
--- a/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationSuiteIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/indices/replication/SegmentReplicationSuiteIT.java
@@ -8,7 +8,6 @@
 
 package org.opensearch.indices.replication;
 
-import org.apache.lucene.tests.util.LuceneTestCase;
 import org.opensearch.action.admin.indices.delete.DeleteIndexRequest;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.common.settings.Settings;
@@ -16,7 +15,6 @@
 import org.opensearch.test.OpenSearchIntegTestCase;
 import org.junit.Before;
 
-@LuceneTestCase.AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/9499")
 @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE, minNumDataNodes = 2)
 public class SegmentReplicationSuiteIT extends SegmentReplicationBaseIT {
 
@@ -64,6 +62,7 @@ public void testDropRandomNodeDuringReplication() throws Exception {
         ensureYellow(INDEX_NAME);
         client().prepareIndex(INDEX_NAME).setId(Integer.toString(docCount)).setSource("field", "value" + docCount).execute().get();
         internalCluster().startDataOnlyNode();
+        ensureGreen(INDEX_NAME);
         client().admin().indices().delete(new DeleteIndexRequest(INDEX_NAME)).actionGet();
     }
 
diff --git a/server/src/internalClusterTest/java/org/opensearch/persistent/PersistentTasksExecutorFullRestartIT.java b/server/src/internalClusterTest/java/org/opensearch/persistent/PersistentTasksExecutorFullRestartIT.java
index 708388b3328f0..151a207d2c191 100644
--- a/server/src/internalClusterTest/java/org/opensearch/persistent/PersistentTasksExecutorFullRestartIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/persistent/PersistentTasksExecutorFullRestartIT.java
@@ -43,6 +43,7 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
+import java.util.concurrent.TimeUnit;
 
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
@@ -129,7 +130,7 @@ public void testFullClusterRestart() throws Exception {
                     .custom(PersistentTasksCustomMetadata.TYPE)).tasks(),
                 empty()
             );
-        });
+        }, 20, TimeUnit.SECONDS);
 
     }
 }
diff --git a/server/src/internalClusterTest/java/org/opensearch/ratelimitting/admissioncontrol/AdmissionForClusterManagerIT.java b/server/src/internalClusterTest/java/org/opensearch/ratelimitting/admissioncontrol/AdmissionForClusterManagerIT.java
index 4d1964326820e..b9da5ffb86af0 100644
--- a/server/src/internalClusterTest/java/org/opensearch/ratelimitting/admissioncontrol/AdmissionForClusterManagerIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/ratelimitting/admissioncontrol/AdmissionForClusterManagerIT.java
@@ -170,8 +170,8 @@ public void testAdmissionControlResponseStatus() throws Exception {
 
             @Override
             public void sendResponse(RestResponse response) {
-                waitForResponse.countDown();
                 aliasResponse.set(response);
+                waitForResponse.countDown();
             }
         };
 
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotemigration/DocRepMigrationTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotemigration/DocRepMigrationTestCase.java
index 5240949ff87b9..61def4ec6e2a4 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotemigration/DocRepMigrationTestCase.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotemigration/DocRepMigrationTestCase.java
@@ -29,7 +29,7 @@ public void testMixedModeAddDocRep() throws Exception {
         ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
         updateSettingsRequest.persistentSettings(Settings.builder().put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed"));
         assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
-        addRemote = false;
+        setAddRemote(false);
         internalCluster().startNode();
         String[] allNodes = internalCluster().getNodeNames();
         assertBusy(() -> { assertEquals(client.admin().cluster().prepareClusterStats().get().getNodes().size(), allNodes.length); });
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotemigration/MigrationBaseTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotemigration/MigrationBaseTestCase.java
index 19da668c432cf..611dfc2756b29 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotemigration/MigrationBaseTestCase.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotemigration/MigrationBaseTestCase.java
@@ -8,17 +8,35 @@
 
 package org.opensearch.remotemigration;
 
+import org.opensearch.action.DocWriteResponse;
 import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesRequest;
 import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesResponse;
+import org.opensearch.action.bulk.BulkRequest;
+import org.opensearch.action.bulk.BulkResponse;
+import org.opensearch.action.delete.DeleteResponse;
+import org.opensearch.action.index.IndexRequest;
+import org.opensearch.action.index.IndexResponse;
+import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.metadata.RepositoryMetadata;
+import org.opensearch.cluster.routing.RoutingNode;
+import org.opensearch.common.UUIDs;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.repositories.fs.ReloadableFsRepository;
 import org.opensearch.test.OpenSearchIntegTestCase;
+import org.junit.Before;
 
 import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicLong;
 
+import static org.opensearch.cluster.routing.allocation.decider.EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING;
+import static org.opensearch.node.remotestore.RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING;
+import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING;
 import static org.opensearch.repositories.fs.ReloadableFsRepository.REPOSITORIES_FAILRATE_SETTING;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
 
@@ -28,8 +46,26 @@ public class MigrationBaseTestCase extends OpenSearchIntegTestCase {
 
     protected Path segmentRepoPath;
     protected Path translogRepoPath;
-
     boolean addRemote = false;
+    Settings extraSettings = Settings.EMPTY;
+
+    private final List<String> documentKeys = List.of(
+        randomAlphaOfLength(5),
+        randomAlphaOfLength(5),
+        randomAlphaOfLength(5),
+        randomAlphaOfLength(5),
+        randomAlphaOfLength(5)
+    );
+
+    void setAddRemote(boolean addRemote) {
+        this.addRemote = addRemote;
+    }
+
+    @Before
+    public void setUp() throws Exception {
+        super.setUp();
+        setAddRemote(false);
+    }
 
     protected Settings nodeSettings(int nodeOrdinal) {
         if (segmentRepoPath == null || translogRepoPath == null) {
@@ -40,6 +76,7 @@ protected Settings nodeSettings(int nodeOrdinal) {
             logger.info("Adding remote store node");
             return Settings.builder()
                 .put(super.nodeSettings(nodeOrdinal))
+                .put(extraSettings)
                 .put(remoteStoreClusterSettings(REPOSITORY_NAME, segmentRepoPath, REPOSITORY_2_NAME, translogRepoPath))
                 .build();
         } else {
@@ -64,4 +101,124 @@ protected void setFailRate(String repoName, int value) throws ExecutionException
             client().admin().cluster().preparePutRepository(repoName).setType(ReloadableFsRepository.TYPE).setSettings(settings).get()
         );
     }
+
+    public void initDocRepToRemoteMigration() {
+        assertTrue(
+            internalCluster().client()
+                .admin()
+                .cluster()
+                .prepareUpdateSettings()
+                .setPersistentSettings(
+                    Settings.builder()
+                        .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed")
+                        .put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store")
+                )
+                .get()
+                .isAcknowledged()
+        );
+    }
+
+    public BulkResponse indexBulk(String indexName, int numDocs) {
+        BulkRequest bulkRequest = new BulkRequest();
+        for (int i = 0; i < numDocs; i++) {
+            final IndexRequest request = client().prepareIndex(indexName)
+                .setId(UUIDs.randomBase64UUID())
+                .setSource(documentKeys.get(randomIntBetween(0, documentKeys.size() - 1)), randomAlphaOfLength(5))
+                .request();
+            bulkRequest.add(request);
+        }
+        return client().bulk(bulkRequest).actionGet();
+    }
+
+    Map<String, Integer> getShardCountByNodeId() {
+        final Map<String, Integer> shardCountByNodeId = new HashMap<>();
+        final ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
+        for (final RoutingNode node : clusterState.getRoutingNodes()) {
+            logger.info(
+                "----> node {} has {} shards",
+                node.nodeId(),
+                clusterState.getRoutingNodes().node(node.nodeId()).numberOfOwningShards()
+            );
+            shardCountByNodeId.put(node.nodeId(), clusterState.getRoutingNodes().node(node.nodeId()).numberOfOwningShards());
+        }
+        return shardCountByNodeId;
+    }
+
+    private void indexSingleDoc(String indexName) {
+        IndexResponse indexResponse = client().prepareIndex(indexName).setId("id").setSource("field", "value").get();
+        assertEquals(DocWriteResponse.Result.CREATED, indexResponse.getResult());
+        DeleteResponse deleteResponse = client().prepareDelete(indexName, "id").get();
+        assertEquals(DocWriteResponse.Result.DELETED, deleteResponse.getResult());
+        client().prepareIndex(indexName).setSource("auto", true).get();
+    }
+
+    public class AsyncIndexingService {
+        private String indexName;
+        private AtomicLong indexedDocs = new AtomicLong(0);
+        private AtomicBoolean finished = new AtomicBoolean();
+        private Thread indexingThread;
+
+        private int refreshFrequency = 3;
+
+        AsyncIndexingService(String indexName) {
+            this.indexName = indexName;
+        }
+
+        public void startIndexing() {
+            indexingThread = getIndexingThread();
+            indexingThread.start();
+        }
+
+        public void stopIndexing() throws InterruptedException {
+            finished.set(true);
+            indexingThread.join();
+        }
+
+        public long getIndexedDocs() {
+            return indexedDocs.get();
+        }
+
+        private Thread getIndexingThread() {
+            return new Thread(() -> {
+                while (finished.get() == false) {
+                    indexSingleDoc(indexName);
+                    long currentDocCount = indexedDocs.incrementAndGet();
+                    if (currentDocCount > 0 && currentDocCount % refreshFrequency == 0) {
+                        logger.info("--> [iteration {}] flushing index", currentDocCount);
+                        if (rarely()) {
+                            client().admin().indices().prepareFlush(indexName).get();
+                        } else {
+                            client().admin().indices().prepareRefresh(indexName).get();
+                        }
+                    }
+                    logger.info("Completed ingestion of {} docs", currentDocCount);
+                }
+            });
+        }
+
+        public void setRefreshFrequency(int refreshFrequency) {
+            this.refreshFrequency = refreshFrequency;
+        }
+    }
+
+    public void excludeNodeSet(String attr, String value) {
+        assertAcked(
+            internalCluster().client()
+                .admin()
+                .cluster()
+                .prepareUpdateSettings()
+                .setTransientSettings(Settings.builder().put("cluster.routing.allocation.exclude._" + attr, value))
+                .get()
+        );
+    }
+
+    public void stopShardRebalancing() {
+        assertAcked(
+            client().admin()
+                .cluster()
+                .prepareUpdateSettings()
+                .setPersistentSettings(Settings.builder().put(CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING.getKey(), "none").build())
+                .get()
+        );
+    }
 }
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteDualReplicationIT.java b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteDualReplicationIT.java
new file mode 100644
index 0000000000000..5094a7cf29c6a
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteDualReplicationIT.java
@@ -0,0 +1,735 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.remotemigration;
+
+import org.opensearch.action.admin.indices.stats.CommonStats;
+import org.opensearch.action.admin.indices.stats.ShardStats;
+import org.opensearch.cluster.ClusterState;
+import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.cluster.node.DiscoveryNodes;
+import org.opensearch.cluster.routing.ShardRouting;
+import org.opensearch.cluster.routing.allocation.command.MoveAllocationCommand;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.index.IndexService;
+import org.opensearch.index.remote.RemoteSegmentStats;
+import org.opensearch.index.seqno.RetentionLease;
+import org.opensearch.index.seqno.RetentionLeases;
+import org.opensearch.indices.IndexingMemoryController;
+import org.opensearch.plugins.Plugin;
+import org.opensearch.remotestore.multipart.mocks.MockFsRepositoryPlugin;
+import org.opensearch.test.InternalSettingsPlugin;
+import org.opensearch.test.InternalTestCluster;
+import org.opensearch.test.OpenSearchIntegTestCase;
+import org.opensearch.test.transport.MockTransportService;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
+
+@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
+public class RemoteDualReplicationIT extends MigrationBaseTestCase {
+    private final String REMOTE_PRI_DOCREP_REP = "remote-primary-docrep-replica";
+    private final String REMOTE_PRI_DOCREP_REMOTE_REP = "remote-primary-docrep-remote-replica";
+    private final String FAILOVER_REMOTE_TO_DOCREP = "failover-remote-to-docrep";
+    private final String FAILOVER_REMOTE_TO_REMOTE = "failover-remote-to-remote";
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        /* Adding the following mock plugins:
+        - InternalSettingsPlugin : To override default intervals of retention lease and global ckp sync
+        - MockFsRepositoryPlugin and MockTransportService.TestPlugin: To ensure remote interactions are not no-op and retention leases are properly propagated
+         */
+        return Stream.concat(
+            super.nodePlugins().stream(),
+            Stream.of(InternalSettingsPlugin.class, MockFsRepositoryPlugin.class, MockTransportService.TestPlugin.class)
+        ).collect(Collectors.toList());
+    }
+
+    /*
+    Scenario:
+    - Starts 2 docrep backed node
+    - Creates index with 1 replica
+    - Index some docs
+    - Start 1 remote backed node
+    - Move primary copy from docrep to remote through _cluster/reroute
+    - Index some more docs
+    - Assert primary-replica consistency
+     */
+    public void testRemotePrimaryDocRepReplica() throws Exception {
+        internalCluster().startClusterManagerOnlyNode();
+
+        logger.info("---> Starting 2 docrep data nodes");
+        internalCluster().startDataOnlyNodes(2);
+        internalCluster().validateClusterFormed();
+        assertEquals(internalCluster().client().admin().cluster().prepareGetRepositories().get().repositories().size(), 0);
+
+        logger.info("---> Creating index with 1 replica");
+        Settings oneReplica = Settings.builder()
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)
+            .put(IndexService.RETENTION_LEASE_SYNC_INTERVAL_SETTING.getKey(), "1s")
+            .put(IndexService.GLOBAL_CHECKPOINT_SYNC_INTERVAL_SETTING.getKey(), "1s")
+            .build();
+        createIndex(REMOTE_PRI_DOCREP_REP, oneReplica);
+        ensureGreen(REMOTE_PRI_DOCREP_REP);
+
+        int initialBatch = randomIntBetween(1, 1000);
+        logger.info("---> Indexing {} docs", initialBatch);
+        indexBulk(REMOTE_PRI_DOCREP_REP, initialBatch);
+
+        initDocRepToRemoteMigration();
+
+        logger.info("---> Starting 1 remote enabled data node");
+        setAddRemote(true);
+        String remoteNodeName = internalCluster().startDataOnlyNode();
+        internalCluster().validateClusterFormed();
+        assertEquals(
+            internalCluster().client()
+                .admin()
+                .cluster()
+                .prepareGetRepositories(REPOSITORY_NAME, REPOSITORY_2_NAME)
+                .get()
+                .repositories()
+                .size(),
+            2
+        );
+
+        String primaryShardHostingNode = primaryNodeName(REMOTE_PRI_DOCREP_REP);
+        logger.info("---> Moving primary copy from {} to remote enabled node {}", primaryShardHostingNode, remoteNodeName);
+        assertAcked(
+            internalCluster().client()
+                .admin()
+                .cluster()
+                .prepareReroute()
+                .add(new MoveAllocationCommand(REMOTE_PRI_DOCREP_REP, 0, primaryShardHostingNode, remoteNodeName))
+                .get()
+        );
+        ensureGreen(REMOTE_PRI_DOCREP_REP);
+        ClusterState clusterState = internalCluster().client().admin().cluster().prepareState().get().getState();
+        String primaryShardHostingNodeId = clusterState.getRoutingTable()
+            .index(REMOTE_PRI_DOCREP_REP)
+            .shard(0)
+            .primaryShard()
+            .currentNodeId();
+        assertTrue(clusterState.getNodes().get(primaryShardHostingNodeId).isRemoteStoreNode());
+
+        int secondBatch = randomIntBetween(1, 10);
+        logger.info("---> Indexing another {} docs", secondBatch);
+        indexBulk(REMOTE_PRI_DOCREP_REP, secondBatch);
+        // Defensive check to ensure that doc count in replica shard catches up to the primary copy
+        refreshAndWaitForReplication(REMOTE_PRI_DOCREP_REP);
+        assertReplicaAndPrimaryConsistency(REMOTE_PRI_DOCREP_REP, initialBatch, secondBatch);
+    }
+
+    /*
+    Scenario:
+    - Starts 2 docrep backed data node
+    - Creates an index with 1 replica
+    - Starts 1 remote backed data node
+    - Index some docs
+    - Move primary copy from docrep to remote through _cluster/reroute
+    - Starts another remote backed data node
+    - Expands index to 2 replicas. One replica copy lies in remote backed node and other in docrep backed node
+    - Index some more docs
+    - Assert primary-replica consistency
+     */
+    public void testRemotePrimaryDocRepAndRemoteReplica() throws Exception {
+        internalCluster().startClusterManagerOnlyNode();
+
+        logger.info("---> Starting 2 docrep data nodes");
+        internalCluster().startDataOnlyNodes(2);
+        internalCluster().validateClusterFormed();
+        assertEquals(internalCluster().client().admin().cluster().prepareGetRepositories().get().repositories().size(), 0);
+
+        logger.info("---> Creating index with 1 replica");
+        Settings zeroReplicas = Settings.builder()
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)
+            .put(IndexService.RETENTION_LEASE_SYNC_INTERVAL_SETTING.getKey(), "1s")
+            .put(IndexService.GLOBAL_CHECKPOINT_SYNC_INTERVAL_SETTING.getKey(), "1s")
+            .build();
+        createIndex(REMOTE_PRI_DOCREP_REMOTE_REP, zeroReplicas);
+        ensureGreen(REMOTE_PRI_DOCREP_REMOTE_REP);
+        initDocRepToRemoteMigration();
+
+        logger.info("---> Starting 1 remote enabled data node");
+        setAddRemote(true);
+
+        String remoteNodeName = internalCluster().startDataOnlyNode();
+        internalCluster().validateClusterFormed();
+        assertEquals(
+            internalCluster().client()
+                .admin()
+                .cluster()
+                .prepareGetRepositories(REPOSITORY_NAME, REPOSITORY_2_NAME)
+                .get()
+                .repositories()
+                .size(),
+            2
+        );
+
+        int firstBatch = randomIntBetween(1, 100);
+        logger.info("---> Indexing {} docs", firstBatch);
+        indexBulk(REMOTE_PRI_DOCREP_REMOTE_REP, firstBatch);
+
+        String primaryShardHostingNode = primaryNodeName(REMOTE_PRI_DOCREP_REMOTE_REP);
+        logger.info("---> Moving primary copy from {} to remote enabled node {}", primaryShardHostingNode, remoteNodeName);
+        assertAcked(
+            internalCluster().client()
+                .admin()
+                .cluster()
+                .prepareReroute()
+                .add(new MoveAllocationCommand(REMOTE_PRI_DOCREP_REMOTE_REP, 0, primaryShardHostingNode, remoteNodeName))
+                .get()
+        );
+        ensureGreen(REMOTE_PRI_DOCREP_REMOTE_REP);
+        ClusterState clusterState = internalCluster().client().admin().cluster().prepareState().get().getState();
+        String primaryShardHostingNodeId = clusterState.getRoutingTable()
+            .index(REMOTE_PRI_DOCREP_REMOTE_REP)
+            .shard(0)
+            .primaryShard()
+            .currentNodeId();
+        assertTrue(clusterState.getNodes().get(primaryShardHostingNodeId).isRemoteStoreNode());
+
+        logger.info("---> Starting another remote enabled node");
+        internalCluster().startDataOnlyNode();
+        internalCluster().validateClusterFormed();
+
+        logger.info("---> Expanding index to 2 replica copies");
+        Settings twoReplicas = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 2).build();
+        assertAcked(
+            internalCluster().client()
+                .admin()
+                .indices()
+                .prepareUpdateSettings()
+                .setIndices(REMOTE_PRI_DOCREP_REMOTE_REP)
+                .setSettings(twoReplicas)
+                .get()
+        );
+        ensureGreen(REMOTE_PRI_DOCREP_REMOTE_REP);
+
+        int secondBatch = randomIntBetween(1, 10);
+        logger.info("---> Indexing another {} docs", secondBatch);
+        indexBulk(REMOTE_PRI_DOCREP_REMOTE_REP, secondBatch);
+        // Defensive check to ensure that doc count in replica shard catches up to the primary copy
+        refreshAndWaitForReplication(REMOTE_PRI_DOCREP_REMOTE_REP);
+        assertReplicaAndPrimaryConsistency(REMOTE_PRI_DOCREP_REMOTE_REP, firstBatch, secondBatch);
+    }
+
+    /*
+    Checks if retention leases are published on primary shard and it's docrep copies, but not on remote copies
+     */
+    public void testRetentionLeasePresentOnDocrepReplicaButNotRemote() throws Exception {
+        /* Reducing indices.memory.shard_inactive_time to force a flush and trigger translog sync,
+        instead of relying on Global CKP Sync action which doesn't run on remote enabled copies
+
+        Under steady state, RetentionLeases would be on (GlobalCkp + 1) on a
+        docrep enabled shard copy and (GlobalCkp) for a remote enabled shard copy.
+        This is because we block translog sync on remote enabled shard copies during the GlobalCkpSync background task.
+
+        RLs on remote enabled copies are brought up to (GlobalCkp + 1) upon a flush request issued by IndexingMemoryController
+        when the shard becomes inactive after SHARD_INACTIVE_TIME_SETTING interval.
+
+        Flush triggers a force sync of translog which bumps the RetentionLease sequence number along with it
+        */
+        extraSettings = Settings.builder().put(IndexingMemoryController.SHARD_INACTIVE_TIME_SETTING.getKey(), "3s").build();
+        testRemotePrimaryDocRepAndRemoteReplica();
+        pollAndCheckRetentionLeases(REMOTE_PRI_DOCREP_REMOTE_REP);
+    }
+
+    /*
+    Scenario:
+    - Starts 2 docrep backed data node
+    - Creates an index with 1 replica
+    - Starts 1 remote backed data node
+    - Index some docs
+    - Move primary copy from docrep to remote through _cluster/reroute
+    - Starts another remote backed data node
+    - Expands index to 2 replicas. One replica copy lies in remote backed node and other in docrep backed node
+    - Index some more docs
+    - Assert retention lease consistency
+     */
+    public void testMissingRetentionLeaseCreatedOnFailedOverRemoteReplica() throws Exception {
+        internalCluster().startClusterManagerOnlyNode();
+
+        logger.info("---> Starting 2 docrep data nodes");
+        internalCluster().startDataOnlyNodes(2);
+
+        Settings zeroReplicasAndOverridenSyncIntervals = Settings.builder()
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)
+            .put(IndexService.GLOBAL_CHECKPOINT_SYNC_INTERVAL_SETTING.getKey(), "100ms")
+            .put(IndexService.RETENTION_LEASE_SYNC_INTERVAL_SETTING.getKey(), "100ms")
+            .build();
+        createIndex(FAILOVER_REMOTE_TO_REMOTE, zeroReplicasAndOverridenSyncIntervals);
+        ensureGreen(FAILOVER_REMOTE_TO_REMOTE);
+
+        indexBulk(FAILOVER_REMOTE_TO_REMOTE, 100);
+
+        logger.info("---> Starting first remote node");
+        initDocRepToRemoteMigration();
+        addRemote = true;
+        String firstRemoteNode = internalCluster().startDataOnlyNode();
+        String primaryShardHostingNode = primaryNodeName(FAILOVER_REMOTE_TO_REMOTE);
+        logger.info("---> Moving primary copy from {} to remote enabled node {}", primaryShardHostingNode, firstRemoteNode);
+        assertAcked(
+            internalCluster().client()
+                .admin()
+                .cluster()
+                .prepareReroute()
+                .add(new MoveAllocationCommand(FAILOVER_REMOTE_TO_REMOTE, 0, primaryShardHostingNode, firstRemoteNode))
+                .get()
+        );
+        ensureGreen(FAILOVER_REMOTE_TO_REMOTE);
+        assertReplicaAndPrimaryConsistency(FAILOVER_REMOTE_TO_REMOTE, 100, 0);
+
+        String secondRemoteNode = internalCluster().startDataOnlyNode();
+        Settings twoReplicas = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 2).build();
+        assertAcked(
+            internalCluster().client()
+                .admin()
+                .indices()
+                .prepareUpdateSettings()
+                .setIndices(FAILOVER_REMOTE_TO_REMOTE)
+                .setSettings(twoReplicas)
+                .get()
+        );
+        ensureGreen(FAILOVER_REMOTE_TO_REMOTE);
+
+        logger.info("---> Checking retention leases");
+        pollAndCheckRetentionLeases(FAILOVER_REMOTE_TO_REMOTE);
+    }
+
+    private void pollAndCheckRetentionLeases(String indexName) throws Exception {
+        DiscoveryNodes nodes = internalCluster().client().admin().cluster().prepareState().get().getState().getNodes();
+        assertBusy(() -> {
+            for (ShardStats shardStats : internalCluster().client().admin().indices().prepareStats(indexName).get().getShards()) {
+                ShardRouting shardRouting = shardStats.getShardRouting();
+                DiscoveryNode discoveryNode = nodes.get(shardRouting.currentNodeId());
+                RetentionLeases retentionLeases = shardStats.getRetentionLeaseStats().retentionLeases();
+                if (shardRouting.primary()) {
+                    // Primary copy should be on remote node and should have retention leases
+                    assertTrue(discoveryNode.isRemoteStoreNode());
+                    assertCheckpointsConsistency(shardStats);
+                    assertRetentionLeaseConsistency(shardStats, retentionLeases);
+                } else {
+                    // Checkpoints and Retention Leases are not synced to remote replicas
+                    if (discoveryNode.isRemoteStoreNode()) {
+                        assertTrue(shardStats.getRetentionLeaseStats().retentionLeases().leases().isEmpty());
+                    } else {
+                        // Replica copy on docrep node should have retention leases
+                        assertCheckpointsConsistency(shardStats);
+                        assertRetentionLeaseConsistency(shardStats, retentionLeases);
+                    }
+                }
+            }
+        });
+    }
+
+    /*
+    Scenario:
+    - Starts 2 docrep backed data node
+    - Creates an index with 1 replica
+    - Starts 1 remote backed data node
+    - Move primary copy from docrep to remote through _cluster/reroute
+    - Stops remote enabled node
+    - Ensure doc count is same after failover
+    - Index some more docs to ensure working of failed-over primary
+     */
+    public void testFailoverRemotePrimaryToDocrepReplica() throws Exception {
+        internalCluster().startClusterManagerOnlyNode();
+
+        logger.info("---> Starting 2 docrep data nodes");
+        internalCluster().startDataOnlyNodes(2);
+        internalCluster().validateClusterFormed();
+        assertEquals(internalCluster().client().admin().cluster().prepareGetRepositories().get().repositories().size(), 0);
+
+        logger.info("---> Creating index with 0 replica");
+        Settings excludeRemoteNode = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).build();
+        createIndex(FAILOVER_REMOTE_TO_DOCREP, excludeRemoteNode);
+        ensureGreen(FAILOVER_REMOTE_TO_DOCREP);
+        initDocRepToRemoteMigration();
+        logger.info("---> Starting 1 remote enabled data node");
+        setAddRemote(true);
+        String remoteNodeName = internalCluster().startDataOnlyNode();
+        internalCluster().validateClusterFormed();
+        assertEquals(
+            internalCluster().client()
+                .admin()
+                .cluster()
+                .prepareGetRepositories(REPOSITORY_NAME, REPOSITORY_2_NAME)
+                .get()
+                .repositories()
+                .size(),
+            2
+        );
+
+        logger.info("---> Starting doc ingestion in parallel thread");
+        AsyncIndexingService asyncIndexingService = new AsyncIndexingService(FAILOVER_REMOTE_TO_DOCREP);
+        asyncIndexingService.startIndexing();
+
+        String primaryShardHostingNode = primaryNodeName(FAILOVER_REMOTE_TO_DOCREP);
+        logger.info("---> Moving primary copy from {} to remote enabled node {}", primaryShardHostingNode, remoteNodeName);
+        assertAcked(
+            internalCluster().client()
+                .admin()
+                .cluster()
+                .prepareReroute()
+                .add(new MoveAllocationCommand(FAILOVER_REMOTE_TO_DOCREP, 0, primaryShardHostingNode, remoteNodeName))
+                .get()
+        );
+        ensureGreen(FAILOVER_REMOTE_TO_DOCREP);
+
+        logger.info("---> Expanding index to 2 replica copies");
+        Settings twoReplicas = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 2).build();
+        assertAcked(
+            internalCluster().client()
+                .admin()
+                .indices()
+                .prepareUpdateSettings()
+                .setIndices(FAILOVER_REMOTE_TO_DOCREP)
+                .setSettings(twoReplicas)
+                .get()
+        );
+        ensureGreen(FAILOVER_REMOTE_TO_DOCREP);
+        logger.info("---> Stopping indexing thread");
+        asyncIndexingService.stopIndexing();
+
+        refreshAndWaitForReplication(FAILOVER_REMOTE_TO_DOCREP);
+        Map<ShardRouting, ShardStats> shardStatsMap = internalCluster().client()
+            .admin()
+            .indices()
+            .prepareStats(FAILOVER_REMOTE_TO_DOCREP)
+            .setDocs(true)
+            .get()
+            .asMap();
+        DiscoveryNodes nodes = internalCluster().client().admin().cluster().prepareState().get().getState().getNodes();
+        long initialPrimaryDocCount = 0;
+        for (ShardRouting shardRouting : shardStatsMap.keySet()) {
+            if (shardRouting.primary()) {
+                assertTrue(nodes.get(shardRouting.currentNodeId()).isRemoteStoreNode());
+                initialPrimaryDocCount = shardStatsMap.get(shardRouting).getStats().getDocs().getCount();
+            }
+        }
+        int firstBatch = (int) asyncIndexingService.getIndexedDocs();
+        assertReplicaAndPrimaryConsistency(FAILOVER_REMOTE_TO_DOCREP, firstBatch, 0);
+
+        logger.info("---> Stop remote store enabled node");
+        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(remoteNodeName));
+        ensureStableCluster(3);
+        ensureYellow(FAILOVER_REMOTE_TO_DOCREP);
+
+        shardStatsMap = internalCluster().client().admin().indices().prepareStats(FAILOVER_REMOTE_TO_DOCREP).setDocs(true).get().asMap();
+        nodes = internalCluster().client().admin().cluster().prepareState().get().getState().getNodes();
+        long primaryDocCountAfterFailover = 0;
+        for (ShardRouting shardRouting : shardStatsMap.keySet()) {
+            if (shardRouting.primary()) {
+                assertFalse(nodes.get(shardRouting.currentNodeId()).isRemoteStoreNode());
+                primaryDocCountAfterFailover = shardStatsMap.get(shardRouting).getStats().getDocs().getCount();
+            }
+        }
+        assertEquals(initialPrimaryDocCount, primaryDocCountAfterFailover);
+
+        logger.info("---> Index some more docs to ensure that the failed over primary is ingesting new docs");
+        int secondBatch = randomIntBetween(1, 10);
+        logger.info("---> Indexing {} more docs", secondBatch);
+        indexBulk(FAILOVER_REMOTE_TO_DOCREP, secondBatch);
+        refreshAndWaitForReplication(FAILOVER_REMOTE_TO_DOCREP);
+
+        shardStatsMap = internalCluster().client().admin().indices().prepareStats(FAILOVER_REMOTE_TO_DOCREP).setDocs(true).get().asMap();
+        assertEquals(2, shardStatsMap.size());
+        shardStatsMap.forEach(
+            (shardRouting, shardStats) -> { assertEquals(firstBatch + secondBatch, shardStats.getStats().getDocs().getCount()); }
+        );
+    }
+
+    /*
+    Scenario:
+    - Starts 2 docrep backed data nodes
+    - Creates an index with 1 replica
+    - Starts 1 remote backed data node
+    - Moves primary copy from docrep to remote through _cluster/reroute
+    - Starts 1 more remote backed data node
+    - Expands index to 2 replicas, one each on new remote node and docrep node
+    - Stops remote enabled node hosting the primary
+    - Ensures remote replica gets promoted to primary
+    - Ensures doc count is same after failover
+    - Indexes some more docs to ensure working of failed-over primary
+    */
+    public void testFailoverRemotePrimaryToRemoteReplica() throws Exception {
+        internalCluster().startClusterManagerOnlyNode();
+
+        logger.info("---> Starting 2 docrep data nodes");
+        List<String> docrepNodeNames = internalCluster().startDataOnlyNodes(2);
+        internalCluster().validateClusterFormed();
+        assertEquals(internalCluster().client().admin().cluster().prepareGetRepositories().get().repositories().size(), 0);
+
+        logger.info("---> Creating index with 1 replica");
+        createIndex(FAILOVER_REMOTE_TO_REMOTE, Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).build());
+        ensureGreen(FAILOVER_REMOTE_TO_REMOTE);
+        initDocRepToRemoteMigration();
+
+        logger.info("---> Starting 1 remote enabled data node");
+        addRemote = true;
+        String remoteNodeName1 = internalCluster().startDataOnlyNode();
+        internalCluster().validateClusterFormed();
+        assertEquals(
+            internalCluster().client()
+                .admin()
+                .cluster()
+                .prepareGetRepositories(REPOSITORY_NAME, REPOSITORY_2_NAME)
+                .get()
+                .repositories()
+                .size(),
+            2
+        );
+
+        logger.info("---> Starting doc ingestion in parallel thread");
+        AsyncIndexingService asyncIndexingService = new AsyncIndexingService(FAILOVER_REMOTE_TO_REMOTE);
+        asyncIndexingService.startIndexing();
+
+        String primaryNodeName = primaryNodeName(FAILOVER_REMOTE_TO_REMOTE);
+        logger.info("---> Moving primary copy from docrep node {} to remote enabled node {}", primaryNodeName, remoteNodeName1);
+        assertAcked(
+            internalCluster().client()
+                .admin()
+                .cluster()
+                .prepareReroute()
+                .add(new MoveAllocationCommand(FAILOVER_REMOTE_TO_REMOTE, 0, primaryNodeName, remoteNodeName1))
+                .get()
+        );
+        waitForRelocation();
+        ensureGreen(FAILOVER_REMOTE_TO_REMOTE);
+        assertEquals(primaryNodeName(FAILOVER_REMOTE_TO_REMOTE), remoteNodeName1);
+
+        logger.info("---> Starting 1 more remote enabled data node");
+        String remoteNodeName2 = internalCluster().startDataOnlyNode();
+        internalCluster().validateClusterFormed();
+
+        logger.info("---> Expanding index to 2 replica copies, on docrepNode and remoteNode2");
+        assertAcked(
+            internalCluster().client()
+                .admin()
+                .indices()
+                .prepareUpdateSettings()
+                .setIndices(FAILOVER_REMOTE_TO_REMOTE)
+                .setSettings(
+                    Settings.builder()
+                        .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 2)
+                        // prevent replica copy from being allocated to the extra docrep node
+                        .put("index.routing.allocation.exclude._name", primaryNodeName)
+                        .build()
+                )
+                .get()
+        );
+        ensureGreen(FAILOVER_REMOTE_TO_REMOTE);
+
+        logger.info("---> Stopping indexing thread");
+        asyncIndexingService.stopIndexing();
+
+        refreshAndWaitForReplication(FAILOVER_REMOTE_TO_REMOTE);
+        Map<ShardRouting, ShardStats> shardStatsMap = internalCluster().client()
+            .admin()
+            .indices()
+            .prepareStats(FAILOVER_REMOTE_TO_REMOTE)
+            .setDocs(true)
+            .get()
+            .asMap();
+        DiscoveryNodes nodes = internalCluster().client().admin().cluster().prepareState().get().getState().getNodes();
+        long initialPrimaryDocCount = 0;
+        for (ShardRouting shardRouting : shardStatsMap.keySet()) {
+            if (shardRouting.primary()) {
+                assertTrue(nodes.get(shardRouting.currentNodeId()).isRemoteStoreNode());
+                initialPrimaryDocCount = shardStatsMap.get(shardRouting).getStats().getDocs().getCount();
+            }
+        }
+        int firstBatch = (int) asyncIndexingService.getIndexedDocs();
+        assertReplicaAndPrimaryConsistency(FAILOVER_REMOTE_TO_REMOTE, firstBatch, 0);
+
+        logger.info("---> Stop remote store enabled node hosting the primary");
+        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(remoteNodeName1));
+        ensureStableCluster(4);
+        ensureYellowAndNoInitializingShards(FAILOVER_REMOTE_TO_REMOTE);
+        DiscoveryNodes finalNodes = internalCluster().client().admin().cluster().prepareState().get().getState().getNodes();
+
+        waitUntil(() -> {
+            ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
+            String nodeId = clusterState.getRoutingTable().index(FAILOVER_REMOTE_TO_REMOTE).shard(0).primaryShard().currentNodeId();
+            if (nodeId == null) {
+                return false;
+            } else {
+                assertEquals(finalNodes.get(nodeId).getName(), remoteNodeName2);
+                return finalNodes.get(nodeId).isRemoteStoreNode();
+            }
+        });
+
+        shardStatsMap = internalCluster().client().admin().indices().prepareStats(FAILOVER_REMOTE_TO_REMOTE).setDocs(true).get().asMap();
+        long primaryDocCountAfterFailover = 0;
+        for (ShardRouting shardRouting : shardStatsMap.keySet()) {
+            if (shardRouting.primary()) {
+                assertTrue(finalNodes.get(shardRouting.currentNodeId()).isRemoteStoreNode());
+                primaryDocCountAfterFailover = shardStatsMap.get(shardRouting).getStats().getDocs().getCount();
+            }
+        }
+        assertEquals(initialPrimaryDocCount, primaryDocCountAfterFailover);
+
+        logger.info("---> Index some more docs to ensure that the failed over primary is ingesting new docs");
+        int secondBatch = randomIntBetween(1, 10);
+        logger.info("---> Indexing {} more docs", secondBatch);
+        indexBulk(FAILOVER_REMOTE_TO_REMOTE, secondBatch);
+        refreshAndWaitForReplication(FAILOVER_REMOTE_TO_REMOTE);
+
+        shardStatsMap = internalCluster().client().admin().indices().prepareStats(FAILOVER_REMOTE_TO_REMOTE).setDocs(true).get().asMap();
+        assertEquals(2, shardStatsMap.size());
+        shardStatsMap.forEach(
+            (shardRouting, shardStats) -> { assertEquals(firstBatch + secondBatch, shardStats.getStats().getDocs().getCount()); }
+        );
+    }
+
+    /*
+    Scenario:
+    - Starts 1 docrep backed data node
+    - Creates an index with 0 replica
+    - Starts 1 remote backed data node
+    - Move primary copy from docrep to remote through _cluster/reroute
+    - Stops remote enabled node
+    - Ensure doc count is same after failover
+    - Index some more docs to ensure working of failed-over primary
+    - Starts another remote node
+    - Move primary copy from docrep to remote through _cluster/reroute
+    - Ensure that remote store is seeded in the new remote node by asserting remote uploads from that node > 0
+     */
+    public void testFailoverRemotePrimaryToDocrepReplicaReseedToRemotePrimary() throws Exception {
+        testFailoverRemotePrimaryToDocrepReplica();
+
+        logger.info("---> Removing replica copy");
+        assertAcked(
+            internalCluster().client()
+                .admin()
+                .indices()
+                .prepareUpdateSettings(FAILOVER_REMOTE_TO_DOCREP)
+                .setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0))
+                .get()
+        );
+        ensureGreen(FAILOVER_REMOTE_TO_DOCREP);
+
+        logger.info("---> Starting a new remote enabled node");
+        setAddRemote(true);
+        String remoteNodeName = internalCluster().startDataOnlyNode();
+        internalCluster().validateClusterFormed();
+        assertEquals(
+            internalCluster().client()
+                .admin()
+                .cluster()
+                .prepareGetRepositories(REPOSITORY_NAME, REPOSITORY_2_NAME)
+                .get()
+                .repositories()
+                .size(),
+            2
+        );
+
+        String primaryShardHostingNode = primaryNodeName(FAILOVER_REMOTE_TO_DOCREP);
+        logger.info("---> Moving primary copy from {} to remote enabled node {}", primaryShardHostingNode, remoteNodeName);
+        assertAcked(
+            internalCluster().client()
+                .admin()
+                .cluster()
+                .prepareReroute()
+                .add(new MoveAllocationCommand(FAILOVER_REMOTE_TO_DOCREP, 0, primaryShardHostingNode, remoteNodeName))
+                .get()
+        );
+        ensureGreen(FAILOVER_REMOTE_TO_DOCREP);
+
+        Map<ShardRouting, ShardStats> shardStatsMap = internalCluster().client()
+            .admin()
+            .indices()
+            .prepareStats(FAILOVER_REMOTE_TO_DOCREP)
+            .get()
+            .asMap();
+        DiscoveryNodes discoveryNodes = internalCluster().client().admin().cluster().prepareState().get().getState().getNodes();
+        assertEquals(1, shardStatsMap.size());
+        shardStatsMap.forEach((shardRouting, shardStats) -> {
+            if (discoveryNodes.get(shardRouting.currentNodeId()).isRemoteStoreNode()) {
+                RemoteSegmentStats remoteSegmentStats = shardStats.getStats().getSegments().getRemoteSegmentStats();
+                assertTrue(remoteSegmentStats.getTotalUploadTime() > 0);
+                assertTrue(remoteSegmentStats.getUploadBytesSucceeded() > 0);
+            }
+        });
+    }
+
+    private void assertReplicaAndPrimaryConsistency(String indexName, int firstBatch, int secondBatch) throws Exception {
+        assertBusy(() -> {
+            Map<ShardRouting, ShardStats> shardStatsMap = internalCluster().client()
+                .admin()
+                .indices()
+                .prepareStats(indexName)
+                .setDocs(true)
+                .get()
+                .asMap();
+            DiscoveryNodes nodes = internalCluster().client().admin().cluster().prepareState().get().getState().getNodes();
+            for (ShardRouting shardRouting : shardStatsMap.keySet()) {
+                CommonStats shardStats = shardStatsMap.get(shardRouting).getStats();
+                if (shardRouting.primary()) {
+                    assertEquals(firstBatch + secondBatch, shardStats.getDocs().getCount());
+                    assertTrue(nodes.get(shardRouting.currentNodeId()).isRemoteStoreNode());
+                    RemoteSegmentStats remoteSegmentStats = shardStats.getSegments().getRemoteSegmentStats();
+                    assertTrue(remoteSegmentStats.getUploadBytesSucceeded() > 0);
+                    assertTrue(remoteSegmentStats.getTotalUploadTime() > 0);
+                }
+                if (shardRouting.unassigned() == false && shardRouting.primary() == false) {
+                    boolean remoteNode = nodes.get(shardRouting.currentNodeId()).isRemoteStoreNode();
+                    assertEquals(
+                        "Mismatched doc count. Is this on remote node ? " + remoteNode,
+                        firstBatch + secondBatch,
+                        shardStats.getDocs().getCount()
+                    );
+                    RemoteSegmentStats remoteSegmentStats = shardStats.getSegments().getRemoteSegmentStats();
+                    if (remoteNode) {
+                        assertTrue(remoteSegmentStats.getDownloadBytesStarted() > 0);
+                        assertTrue(remoteSegmentStats.getTotalDownloadTime() > 0);
+                    } else {
+                        assertEquals(0, remoteSegmentStats.getUploadBytesSucceeded());
+                        assertEquals(0, remoteSegmentStats.getTotalUploadTime());
+                    }
+                }
+            }
+        });
+    }
+
+    /**
+     * For a docrep enabled shard copy or a primary shard copy,
+     * asserts that the stored Retention Leases equals to 1 + maxSeqNo ingested on the node
+     *
+     * @param shardStats ShardStats object from NodesStats API
+     * @param retentionLeases RetentionLeases from NodesStats API
+     */
+    private static void assertRetentionLeaseConsistency(ShardStats shardStats, RetentionLeases retentionLeases) {
+        long maxSeqNo = shardStats.getSeqNoStats().getMaxSeqNo();
+        for (RetentionLease rl : retentionLeases.leases()) {
+            assertEquals(maxSeqNo + 1, rl.retainingSequenceNumber());
+        }
+    }
+
+    /**
+     * For a docrep enabled shard copy or a primary shard copy,
+     * asserts that local and global checkpoints are up-to-date with maxSeqNo of doc operations
+     *
+     * @param shardStats ShardStats object from NodesStats API
+     */
+    private static void assertCheckpointsConsistency(ShardStats shardStats) {
+        long maxSeqNo = shardStats.getSeqNoStats().getMaxSeqNo();
+        long localCkp = shardStats.getSeqNoStats().getLocalCheckpoint();
+        long globalCkp = shardStats.getSeqNoStats().getGlobalCheckpoint();
+
+        assertEquals(maxSeqNo, localCkp);
+        assertEquals(maxSeqNo, globalCkp);
+    }
+}
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteMigrationAllocationDeciderIT.java b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteMigrationAllocationDeciderIT.java
new file mode 100644
index 0000000000000..eeb6a5a5626e4
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteMigrationAllocationDeciderIT.java
@@ -0,0 +1,537 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.remotemigration;
+
+import org.opensearch.action.admin.cluster.health.ClusterHealthResponse;
+import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
+import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.cluster.routing.ShardRouting;
+import org.opensearch.cluster.routing.UnassignedInfo;
+import org.opensearch.cluster.routing.allocation.command.MoveAllocationCommand;
+import org.opensearch.cluster.routing.allocation.decider.Decision;
+import org.opensearch.common.Priority;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.TimeValue;
+import org.opensearch.test.InternalTestCluster;
+import org.opensearch.test.OpenSearchIntegTestCase;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Locale;
+import java.util.Optional;
+
+import static org.opensearch.node.remotestore.RemoteStoreNodeService.CompatibilityMode.MIXED;
+import static org.opensearch.node.remotestore.RemoteStoreNodeService.Direction.REMOTE_STORE;
+import static org.opensearch.node.remotestore.RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING;
+import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING;
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
+
+@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
+public class RemoteMigrationAllocationDeciderIT extends RemoteStoreMigrationShardAllocationBaseTestCase {
+
+    // When the primary is on doc rep node, existing replica copy can get allocated on excluded docrep node.
+    public void testFilterAllocationSkipsReplica() throws IOException {
+        addRemote = false;
+        List<String> docRepNodes = internalCluster().startNodes(3);
+        createIndex(
+            "test",
+            Settings.builder()
+                .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)
+                .put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "0")
+                .build()
+        );
+        ensureGreen("test");
+
+        ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
+        updateSettingsRequest.persistentSettings(
+            Settings.builder()
+                .put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store")
+                .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed")
+        );
+        assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
+
+        assertTrue(
+            internalCluster().client()
+                .admin()
+                .indices()
+                .prepareUpdateSettings("test")
+                .setSettings(Settings.builder().put("index.routing.allocation.exclude._name", String.join(",", docRepNodes)))
+                .execute()
+                .actionGet()
+                .isAcknowledged()
+        );
+        internalCluster().stopRandomDataNode();
+        ensureGreen("test");
+    }
+
+    // When the primary is on remote node, new replica copy shouldn't get allocated on an excluded docrep node.
+    public void testFilterAllocationSkipsReplicaOnExcludedNode() throws IOException {
+        addRemote = false;
+        List<String> nodes = internalCluster().startNodes(2);
+        createIndex(
+            "test",
+            Settings.builder()
+                .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)
+                .put(UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING.getKey(), "0")
+                .build()
+        );
+        ensureGreen("test");
+        addRemote = true;
+
+        ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
+        updateSettingsRequest.persistentSettings(
+            Settings.builder()
+                .put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store")
+                .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed")
+        );
+        assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
+        String remoteNode = internalCluster().startNode();
+
+        client().admin()
+            .cluster()
+            .prepareReroute()
+            .add(new MoveAllocationCommand("test", 0, primaryNodeName("test"), remoteNode))
+            .execute()
+            .actionGet();
+        client().admin()
+            .cluster()
+            .prepareHealth()
+            .setTimeout(TimeValue.timeValueSeconds(60))
+            .setWaitForEvents(Priority.LANGUID)
+            .setWaitForNoRelocatingShards(true)
+            .execute()
+            .actionGet();
+        assertEquals(remoteNode, primaryNodeName("test"));
+
+        assertTrue(
+            internalCluster().client()
+                .admin()
+                .indices()
+                .prepareUpdateSettings("test")
+                .setSettings(Settings.builder().put("index.routing.allocation.exclude._name", String.join(",", nodes)))
+                .execute()
+                .actionGet()
+                .isAcknowledged()
+        );
+        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(replicaNodeName("test")));
+        ClusterHealthResponse clusterHealthResponse = client().admin()
+            .cluster()
+            .prepareHealth()
+            .setWaitForEvents(Priority.LANGUID)
+            .setWaitForGreenStatus()
+            .setTimeout(TimeValue.timeValueSeconds(2))
+            .execute()
+            .actionGet();
+        assertTrue(clusterHealthResponse.isTimedOut());
+        ensureYellow("test");
+    }
+
+    // When under mixed mode and remote_store direction, a primary shard can only be allocated to a remote node
+
+    public void testNewPrimaryShardAllocationForRemoteStoreMigration() throws Exception {
+        logger.info("Initialize cluster");
+        internalCluster().startClusterManagerOnlyNode();
+
+        logger.info("Add non-remote data node");
+        String nonRemoteNodeName = internalCluster().startDataOnlyNode();
+        internalCluster().validateClusterFormed();
+        DiscoveryNode nonRemoteNode = assertNodeInCluster(nonRemoteNodeName);
+
+        logger.info("Set mixed mode and remote_store direction");
+        setClusterMode(MIXED.mode);
+        setDirection(REMOTE_STORE.direction);
+
+        logger.info("Verify expected decision for allocating a new primary shard on a non-remote node");
+        prepareIndexWithoutReplica(Optional.empty());
+        Decision decision = getDecisionForTargetNode(nonRemoteNode, true, true, false);
+        assertEquals(Decision.Type.NO, decision.type());
+        assertEquals(
+            "[remote_store migration_direction]: primary shard copy can not be allocated to a non-remote node",
+            decision.getExplanation().toLowerCase(Locale.ROOT)
+        );
+
+        logger.info("Attempt allocation on non-remote node");
+        attemptAllocation(null);
+
+        logger.info("Verify non-allocation of primary shard on non-remote node");
+        assertNonAllocation(true);
+
+        logger.info("Add remote data node");
+        setAddRemote(true);
+        String remoteNodeName = internalCluster().startDataOnlyNode();
+        internalCluster().validateClusterFormed();
+        DiscoveryNode remoteNode = assertNodeInCluster(remoteNodeName);
+
+        logger.info("Verify expected decision for allocating a new primary shard on a remote node");
+        excludeAllNodes();
+        decision = getDecisionForTargetNode(remoteNode, true, true, false);
+        assertEquals(Decision.Type.YES, decision.type());
+        assertEquals(
+            "[remote_store migration_direction]: primary shard copy can be allocated to a remote node",
+            decision.getExplanation().toLowerCase(Locale.ROOT)
+        );
+
+        logger.info("Attempt free allocation");
+        attemptAllocation(null);
+        ensureGreen(TEST_INDEX);
+
+        logger.info("Verify allocation of primary shard on remote node");
+        assertAllocation(true, remoteNode);
+    }
+
+    // When under mixed mode and remote_store direction, a replica shard can only be allocated to a remote node if the primary has relocated
+    // to another remote node
+
+    public void testNewReplicaShardAllocationIfPrimaryShardOnNonRemoteNodeForRemoteStoreMigration() throws Exception {
+        logger.info("Initialize cluster");
+        internalCluster().startClusterManagerOnlyNode();
+
+        logger.info("Add non-remote data node");
+        String nonRemoteNodeName1 = internalCluster().startDataOnlyNode();
+        internalCluster().validateClusterFormed();
+        DiscoveryNode nonRemoteNode1 = assertNodeInCluster(nonRemoteNodeName1);
+
+        logger.info("Allocate primary shard on non-remote node");
+        prepareIndexWithAllocatedPrimary(nonRemoteNode1, Optional.empty());
+
+        logger.info("Add remote data node");
+        setClusterMode(MIXED.mode);
+        setAddRemote(true);
+        String remoteNodeName = internalCluster().startDataOnlyNode();
+        internalCluster().validateClusterFormed();
+        DiscoveryNode remoteNode = assertNodeInCluster(remoteNodeName);
+
+        logger.info("Set remote_store direction");
+        setDirection(REMOTE_STORE.direction);
+
+        logger.info("Verify expected decision for allocating a replica shard on a remote node");
+        excludeAllNodes();
+        Decision decision = getDecisionForTargetNode(remoteNode, false, true, false);
+        assertEquals(Decision.Type.NO, decision.type());
+        assertEquals(
+            "[remote_store migration_direction]: replica shard copy can not be allocated to a remote node since primary shard copy is not yet migrated to remote",
+            decision.getExplanation().toLowerCase(Locale.ROOT)
+        );
+
+        logger.info("Attempt free allocation of replica shard");
+        attemptAllocation(null);
+
+        logger.info("Verify non-allocation of replica shard");
+        assertNonAllocation(false);
+
+        logger.info("Add another non-remote data node");
+        setAddRemote(false);
+        String nonRemoteNodeName2 = internalCluster().startDataOnlyNode();
+        internalCluster().validateClusterFormed();
+        DiscoveryNode nonRemoteNode2 = assertNodeInCluster(nonRemoteNodeName2);
+
+        logger.info("Verify expected decision for allocating the replica shard on a non-remote node");
+        excludeAllNodes();
+        decision = getDecisionForTargetNode(nonRemoteNode2, false, true, false);
+        assertEquals(Decision.Type.YES, decision.type());
+        assertEquals(
+            "[remote_store migration_direction]: replica shard copy can be allocated to a non-remote node",
+            decision.getExplanation().toLowerCase(Locale.ROOT)
+        );
+
+        logger.info("Attempt free allocation of replica shard");
+        attemptAllocation(null);
+        ensureGreen(TEST_INDEX);
+
+        logger.info("Verify allocation of replica shard on non-remote node");
+        assertAllocation(false, nonRemoteNode2);
+    }
+
+    public void testNewReplicaShardAllocationIfPrimaryShardOnRemoteNodeForRemoteStoreMigration() throws Exception {
+        logger.info("Initialize cluster");
+        internalCluster().startClusterManagerOnlyNode();
+
+        logger.info("Add non-remote data nodes");
+        String nonRemoteNodeName1 = internalCluster().startDataOnlyNode();
+        String nonRemoteNodeName2 = internalCluster().startDataOnlyNode();
+        internalCluster().validateClusterFormed();
+        DiscoveryNode nonRemoteNode1 = assertNodeInCluster(nonRemoteNodeName1);
+        DiscoveryNode nonRemoteNode2 = assertNodeInCluster(nonRemoteNodeName2);
+
+        logger.info("Allocate primary and replica shard on non-remote nodes");
+        createIndex(TEST_INDEX, 1);
+        ensureGreen(TEST_INDEX);
+
+        logger.info("Set mixed mode");
+        setClusterMode(MIXED.mode);
+
+        logger.info("Add remote data nodes");
+        setAddRemote(true);
+        String remoteNodeName1 = internalCluster().startDataOnlyNode();
+        String remoteNodeName2 = internalCluster().startDataOnlyNode();
+        internalCluster().validateClusterFormed();
+        DiscoveryNode remoteNode1 = assertNodeInCluster(remoteNodeName1);
+        DiscoveryNode remoteNode2 = assertNodeInCluster(remoteNodeName2);
+
+        logger.info("Set remote_store direction");
+        setDirection(REMOTE_STORE.direction);
+
+        logger.info("Relocate primary shard to remote node");
+        DiscoveryNode initialPrimaryNode = primaryNodeName(TEST_INDEX).equals(nonRemoteNodeName1) ? nonRemoteNode1 : nonRemoteNode2;
+        DiscoveryNode initialReplicaNode = initialPrimaryNode.equals(nonRemoteNode1) ? nonRemoteNode2 : nonRemoteNode1;
+        assertAcked(
+            internalCluster().client()
+                .admin()
+                .cluster()
+                .prepareReroute()
+                .add(new MoveAllocationCommand(TEST_INDEX, 0, initialPrimaryNode.getName(), remoteNodeName1))
+                .get()
+        );
+        ensureGreen(TEST_INDEX);
+        assertAllocation(true, remoteNode1);
+
+        logger.info("Verify expected decision for relocating a replica shard on non-remote node");
+        Decision decision = getDecisionForTargetNode(initialPrimaryNode, false, true, true);
+        assertEquals(Decision.Type.YES, decision.type());
+        assertEquals(
+            "[remote_store migration_direction]: replica shard copy can be relocated to a non-remote node",
+            decision.getExplanation().toLowerCase(Locale.ROOT)
+        );
+
+        logger.info("Attempt relocation of replica shard to non-remote node");
+        assertAcked(
+            internalCluster().client()
+                .admin()
+                .cluster()
+                .prepareReroute()
+                .add(new MoveAllocationCommand(TEST_INDEX, 0, initialReplicaNode.getName(), initialPrimaryNode.getName()))
+                .get()
+        );
+
+        logger.info("Verify relocation of replica shard to non-remote node");
+        ensureGreen(TEST_INDEX);
+        assertAllocation(false, initialPrimaryNode);
+
+        logger.info("Verify expected decision for relocating a replica shard on remote node");
+        decision = getDecisionForTargetNode(remoteNode2, false, true, true);
+        assertEquals(Decision.Type.YES, decision.type());
+        assertEquals(
+            "[remote_store migration_direction]: replica shard copy can be relocated to a remote node since primary shard copy has been migrated to remote",
+            decision.getExplanation().toLowerCase(Locale.ROOT)
+        );
+
+        logger.info("Attempt relocation of replica shard to remote node");
+        assertAcked(
+            internalCluster().client()
+                .admin()
+                .cluster()
+                .prepareReroute()
+                .add(new MoveAllocationCommand(TEST_INDEX, 0, initialPrimaryNode.getName(), remoteNodeName2))
+                .get()
+        );
+
+        logger.info("Verify relocation of replica shard to non-remote node");
+        ensureGreen(TEST_INDEX);
+        assertAllocation(false, remoteNode2);
+    }
+
+    // When under strict mode, a shard can be allocated to any node
+
+    public void testAlwaysAllocateNewShardForStrictMode() throws Exception {
+        boolean isRemoteCluster = randomBoolean();
+        boolean isReplicaAllocation = randomBoolean();
+
+        logger.info("Initialize cluster and add nodes");
+        setAddRemote(isRemoteCluster);
+        internalCluster().startClusterManagerOnlyNode();
+        String nodeName1 = internalCluster().startDataOnlyNode();
+        String nodeName2 = internalCluster().startDataOnlyNode();
+        internalCluster().validateClusterFormed();
+        DiscoveryNode node1 = assertNodeInCluster(nodeName1);
+        DiscoveryNode node2 = assertNodeInCluster(nodeName2);
+
+        if (isReplicaAllocation) {
+            prepareIndexWithAllocatedPrimary(node1, Optional.empty());
+        } else {
+            prepareIndexWithoutReplica(Optional.empty());
+        }
+
+        if (isRemoteCluster) {
+            assertRemoteStoreBackedIndex(TEST_INDEX);
+        } else {
+            assertNonRemoteStoreBackedIndex(TEST_INDEX);
+        }
+
+        logger.info("Verify expected decision for allocation of a shard");
+        excludeAllNodes();
+        Decision decision = getDecisionForTargetNode(
+            isReplicaAllocation ? node2 : randomFrom(node1, node2),
+            !isReplicaAllocation,
+            true,
+            false
+        );
+        assertEquals(Decision.Type.YES, decision.type());
+        String expectedReason = String.format(
+            Locale.ROOT,
+            "[none migration_direction]: %s shard copy can be allocated to a %s node for strict compatibility mode",
+            (isReplicaAllocation ? "replica" : "primary"),
+            (isRemoteCluster ? "remote" : "non-remote")
+        );
+        assertEquals(expectedReason, decision.getExplanation().toLowerCase(Locale.ROOT));
+
+        logger.info("Attempt free allocation");
+        attemptAllocation(null);
+        ensureGreen(TEST_INDEX);
+
+        logger.info("Verify allocation of shard");
+        assertAllocation(!isReplicaAllocation, !isReplicaAllocation ? null : node2);
+    }
+
+    // When under mixed mode and remote_store direction, shard of a remote store backed index can not be allocated to a non-remote node
+
+    public void testRemoteStoreBackedIndexShardAllocationForRemoteStoreMigration() throws Exception {
+        logger.info("Initialize cluster");
+        internalCluster().startClusterManagerOnlyNode();
+
+        logger.info("Set mixed mode");
+        setClusterMode(MIXED.mode);
+
+        logger.info("Add remote and non-remote nodes");
+        String nonRemoteNodeName = internalCluster().startDataOnlyNode();
+        setAddRemote(true);
+        String remoteNodeName = internalCluster().startDataOnlyNode();
+        internalCluster().validateClusterFormed();
+        DiscoveryNode remoteNode = assertNodeInCluster(remoteNodeName);
+        DiscoveryNode nonRemoteNode = assertNodeInCluster(nonRemoteNodeName);
+
+        logger.info("Set remote_store direction");
+        setDirection(REMOTE_STORE.direction);
+
+        boolean isReplicaAllocation = randomBoolean();
+        if (isReplicaAllocation) {
+            logger.info("Create index with primary allocated on remote node");
+            prepareIndexWithAllocatedPrimary(remoteNode, Optional.empty());
+        } else {
+            logger.info("Create index with unallocated primary");
+            prepareIndexWithoutReplica(Optional.empty());
+        }
+
+        logger.info("Verify remote store backed index");
+        assertRemoteStoreBackedIndex(TEST_INDEX);
+
+        logger.info("Verify expected decision for allocation of shard on a non-remote node");
+        excludeAllNodes();
+        Decision decision = getDecisionForTargetNode(nonRemoteNode, !isReplicaAllocation, false, false);
+        assertEquals(Decision.Type.NO, decision.type());
+        String expectedReason = String.format(
+            Locale.ROOT,
+            "[remote_store migration_direction]: %s shard copy can not be allocated to a non-remote node because a remote store backed index's shard copy can only be allocated to a remote node",
+            (isReplicaAllocation ? "replica" : "primary")
+        );
+        assertEquals(expectedReason, decision.getExplanation().toLowerCase(Locale.ROOT));
+
+        logger.info("Attempt allocation of shard on non-remote node");
+        attemptAllocation(nonRemoteNodeName);
+
+        logger.info("Verify non-allocation of shard");
+        assertNonAllocation(!isReplicaAllocation);
+    }
+
+    // When under mixed mode and none direction, allocate shard of a remote store backed index to a remote node and shard of a non remote
+    // store backed index to a non-remote node only
+
+    public void testAllocationForNoneDirectionAndMixedMode() throws Exception {
+        boolean isRemoteStoreBackedIndex = randomBoolean();
+        boolean isReplicaAllocation = randomBoolean();
+        logger.info(
+            String.format(
+                Locale.ROOT,
+                "Test for allocation decisions for %s shard of a %s store backed index under NONE direction",
+                (isReplicaAllocation ? "replica" : "primary"),
+                (isRemoteStoreBackedIndex ? "remote" : "non remote")
+            )
+        );
+
+        logger.info("Initialize cluster");
+        setAddRemote(isRemoteStoreBackedIndex);
+        internalCluster().startClusterManagerOnlyNode();
+
+        logger.info("Add data nodes");
+        String previousNodeName1 = internalCluster().startDataOnlyNode();
+        String previousNodeName2 = internalCluster().startDataOnlyNode();
+        internalCluster().validateClusterFormed();
+        DiscoveryNode previousNode1 = assertNodeInCluster(previousNodeName1);
+        DiscoveryNode previousNode2 = assertNodeInCluster(previousNodeName2);
+
+        logger.info("Prepare test index");
+        if (isReplicaAllocation) {
+            prepareIndexWithAllocatedPrimary(previousNode1, Optional.empty());
+        } else {
+            prepareIndexWithoutReplica(Optional.empty());
+        }
+
+        if (isRemoteStoreBackedIndex) {
+            assertRemoteStoreBackedIndex(TEST_INDEX);
+        } else {
+            assertNonRemoteStoreBackedIndex(TEST_INDEX);
+        }
+
+        logger.info("Switch to MIXED cluster compatibility mode");
+        setClusterMode(MIXED.mode);
+        setAddRemote(!addRemote);
+        String newNodeName = internalCluster().startDataOnlyNode();
+        internalCluster().validateClusterFormed();
+        DiscoveryNode newNode = assertNodeInCluster(newNodeName);
+
+        logger.info("Verify decision for allocation on the new node");
+        excludeAllNodes();
+        Decision decision = getDecisionForTargetNode(newNode, !isReplicaAllocation, false, false);
+        assertEquals(Decision.Type.NO, decision.type());
+        String expectedReason = String.format(
+            Locale.ROOT,
+            "[none migration_direction]: %s shard copy can not be allocated to a %s node for %s store backed index",
+            (isReplicaAllocation ? "replica" : "primary"),
+            (isRemoteStoreBackedIndex ? "non-remote" : "remote"),
+            (isRemoteStoreBackedIndex ? "remote" : "non remote")
+        );
+        assertEquals(expectedReason, decision.getExplanation().toLowerCase(Locale.ROOT));
+
+        logger.info("Attempt allocation of shard on new node");
+        attemptAllocation(newNodeName);
+
+        logger.info("Verify non-allocation of shard");
+        assertNonAllocation(!isReplicaAllocation);
+
+        logger.info("Verify decision for allocation on previous node");
+        decision = getDecisionForTargetNode(previousNode2, !isReplicaAllocation, true, false);
+        assertEquals(Decision.Type.YES, decision.type());
+        expectedReason = String.format(
+            Locale.ROOT,
+            "[none migration_direction]: %s shard copy can be allocated to a %s node for %s store backed index",
+            (isReplicaAllocation ? "replica" : "primary"),
+            (isRemoteStoreBackedIndex ? "remote" : "non-remote"),
+            (isRemoteStoreBackedIndex ? "remote" : "non remote")
+        );
+        assertEquals(expectedReason, decision.getExplanation().toLowerCase(Locale.ROOT));
+
+        logger.info("Attempt free allocation of shard");
+        attemptAllocation(null);
+
+        logger.info("Verify successful allocation of shard");
+        if (!isReplicaAllocation) {
+            ensureGreen(TEST_INDEX);
+        } else {
+            ensureYellowAndNoInitializingShards(TEST_INDEX);
+        }
+        assertAllocation(!isReplicaAllocation, null);
+        logger.info("Verify allocation on one of the previous nodes");
+        ShardRouting shardRouting = getShardRouting(!isReplicaAllocation);
+        assertTrue(
+            shardRouting.currentNodeId().equals(previousNode1.getId()) || shardRouting.currentNodeId().equals(previousNode2.getId())
+        );
+    }
+}
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteMigrationIndexMetadataUpdateIT.java b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteMigrationIndexMetadataUpdateIT.java
new file mode 100644
index 0000000000000..45679598dc551
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteMigrationIndexMetadataUpdateIT.java
@@ -0,0 +1,516 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.remotemigration;
+
+import org.opensearch.cluster.ClusterState;
+import org.opensearch.cluster.health.ClusterHealthStatus;
+import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.cluster.node.DiscoveryNodes;
+import org.opensearch.cluster.routing.ShardRouting;
+import org.opensearch.cluster.routing.allocation.command.MoveAllocationCommand;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.indices.replication.common.ReplicationType;
+import org.opensearch.test.InternalTestCluster;
+import org.opensearch.test.OpenSearchIntegTestCase;
+
+import java.util.List;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
+
+@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
+public class RemoteMigrationIndexMetadataUpdateIT extends MigrationBaseTestCase {
+    /**
+     * Scenario:
+     * Performs a blue/green type migration from docrep to remote enabled cluster.
+     * Asserts that remote based index settings are applied after all shards move over
+     */
+    public void testIndexSettingsUpdateAfterIndexMovedToRemoteThroughAllocationExclude() throws Exception {
+        internalCluster().startClusterManagerOnlyNode();
+
+        logger.info("---> Starting 2 docrep nodes");
+        addRemote = false;
+        internalCluster().startDataOnlyNodes(2, Settings.builder().put("node.attr._type", "docrep").build());
+        internalCluster().validateClusterFormed();
+
+        logger.info("---> Creates an index with 1 primary and 1 replica");
+        String indexName = "migration-index-allocation-exclude";
+        Settings oneReplica = Settings.builder()
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+            .build();
+        logger.info("---> Asserts index still has docrep index settings");
+        createIndexAndAssertDocrepProperties(indexName, oneReplica);
+
+        logger.info("---> Start indexing in parallel thread");
+        AsyncIndexingService asyncIndexingService = new AsyncIndexingService(indexName);
+        asyncIndexingService.startIndexing();
+        initDocRepToRemoteMigration();
+
+        logger.info("---> Adding 2 remote enabled nodes to the cluster");
+        addRemote = true;
+        internalCluster().startDataOnlyNodes(2, Settings.builder().put("node.attr._type", "remote").build());
+        internalCluster().validateClusterFormed();
+
+        logger.info("---> Excluding docrep nodes from allocation");
+        excludeNodeSet("type", "docrep");
+        waitForRelocation();
+        waitNoPendingTasksOnAll();
+
+        logger.info("---> Stop indexing and assert remote enabled index settings have been applied");
+        asyncIndexingService.stopIndexing();
+        assertRemoteProperties(indexName);
+    }
+
+    /**
+     * Scenario:
+     * Performs a manual _cluster/reroute to move shards from docrep to remote enabled nodes.
+     * Asserts that remote based index settings are only applied for indices whose shards
+     * have completely moved over to remote enabled nodes
+     */
+    public void testIndexSettingsUpdateAfterIndexMovedToRemoteThroughManualReroute() throws Exception {
+        internalCluster().startClusterManagerOnlyNode();
+
+        logger.info("---> Starting 2 docrep nodes");
+        List<String> docrepNodeNames = internalCluster().startDataOnlyNodes(2);
+        internalCluster().validateClusterFormed();
+
+        logger.info("---> Creating 2 indices with 1 primary and 1 replica");
+        String indexName1 = "migration-index-manual-reroute-1";
+        String indexName2 = "migration-index-manual-reroute-2";
+        Settings oneReplica = Settings.builder()
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+            .build();
+        createIndexAndAssertDocrepProperties(indexName1, oneReplica);
+        createIndexAndAssertDocrepProperties(indexName2, oneReplica);
+
+        logger.info("---> Starting parallel indexing on both indices");
+        AsyncIndexingService indexOne = new AsyncIndexingService(indexName1);
+        indexOne.startIndexing();
+
+        AsyncIndexingService indexTwo = new AsyncIndexingService(indexName2);
+        indexTwo.startIndexing();
+
+        logger.info(
+            "---> Stopping shard rebalancing to ensure shards do not automatically move over to newer nodes after they are launched"
+        );
+        stopShardRebalancing();
+
+        logger.info("---> Starting 2 remote store enabled nodes");
+        initDocRepToRemoteMigration();
+        addRemote = true;
+        List<String> remoteNodeNames = internalCluster().startDataOnlyNodes(2);
+        internalCluster().validateClusterFormed();
+
+        String primaryNode = primaryNodeName(indexName1);
+        String replicaNode = docrepNodeNames.stream()
+            .filter(nodeName -> nodeName.equals(primaryNodeName(indexName1)) == false)
+            .collect(Collectors.toList())
+            .get(0);
+
+        logger.info("---> Moving over both shard copies for the first index to remote enabled nodes");
+        assertAcked(
+            client().admin()
+                .cluster()
+                .prepareReroute()
+                .add(new MoveAllocationCommand(indexName1, 0, primaryNode, remoteNodeNames.get(0)))
+                .execute()
+                .actionGet()
+        );
+        waitForRelocation();
+
+        assertAcked(
+            client().admin()
+                .cluster()
+                .prepareReroute()
+                .add(new MoveAllocationCommand(indexName1, 0, replicaNode, remoteNodeNames.get(1)))
+                .execute()
+                .actionGet()
+        );
+        waitForRelocation();
+
+        logger.info("---> Moving only primary for the second index to remote enabled nodes");
+        assertAcked(
+            client().admin()
+                .cluster()
+                .prepareReroute()
+                .add(new MoveAllocationCommand(indexName2, 0, primaryNodeName(indexName2), remoteNodeNames.get(0)))
+                .execute()
+                .actionGet()
+        );
+        waitForRelocation();
+        waitNoPendingTasksOnAll();
+
+        logger.info("---> Stopping indexing");
+        indexOne.stopIndexing();
+        indexTwo.stopIndexing();
+
+        logger.info("---> Assert remote settings are applied for index one but not for index two");
+        assertRemoteProperties(indexName1);
+        assertDocrepProperties(indexName2);
+    }
+
+    /**
+     * Scenario:
+     * Creates a mixed mode cluster. One index gets created before remote nodes are introduced,
+     * while the other one is created after remote nodes are added.
+     * <p>
+     * For the first index, asserts docrep settings at first, excludes docrep nodes from
+     * allocation and asserts that remote index settings are applied after all shards
+     * have been relocated.
+     * <p>
+     * For the second index, asserts that it already has remote enabled settings.
+     * Indexes some more docs and asserts that the index metadata version does not increment
+     */
+    public void testIndexSettingsUpdatedOnlyForMigratingIndex() throws Exception {
+        internalCluster().startClusterManagerOnlyNode();
+
+        logger.info("---> Starting 2 docrep nodes");
+        addRemote = false;
+        internalCluster().startDataOnlyNodes(2, Settings.builder().put("node.attr._type", "docrep").build());
+        internalCluster().validateClusterFormed();
+
+        logger.info("---> Creating the first index with 1 primary and 1 replica");
+        String indexName = "migration-index";
+        Settings oneReplica = Settings.builder()
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+            .build();
+        createIndexAndAssertDocrepProperties(indexName, oneReplica);
+
+        logger.info("---> Starting indexing in parallel");
+        AsyncIndexingService indexingService = new AsyncIndexingService(indexName);
+        indexingService.startIndexing();
+
+        logger.info("---> Storing current index metadata version");
+        long initalMetadataVersion = internalCluster().client()
+            .admin()
+            .cluster()
+            .prepareState()
+            .get()
+            .getState()
+            .metadata()
+            .index(indexName)
+            .getVersion();
+
+        logger.info("---> Adding 2 remote enabled nodes to the cluster");
+        initDocRepToRemoteMigration();
+        addRemote = true;
+        internalCluster().startDataOnlyNodes(2, Settings.builder().put("node.attr._type", "remote").build());
+        internalCluster().validateClusterFormed();
+
+        logger.info("---> Excluding docrep nodes from allocation");
+        excludeNodeSet("type", "docrep");
+
+        waitForRelocation();
+        waitNoPendingTasksOnAll();
+        indexingService.stopIndexing();
+
+        logger.info("---> Assert remote settings are applied");
+        assertRemoteProperties(indexName);
+        assertTrue(
+            initalMetadataVersion < internalCluster().client()
+                .admin()
+                .cluster()
+                .prepareState()
+                .get()
+                .getState()
+                .metadata()
+                .index(indexName)
+                .getVersion()
+        );
+
+        logger.info("---> Creating a new index on remote enabled nodes");
+        String secondIndex = "remote-index";
+        createIndex(
+            secondIndex,
+            Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).build()
+        );
+        indexBulk(secondIndex, 100);
+        initalMetadataVersion = internalCluster().client()
+            .admin()
+            .cluster()
+            .prepareState()
+            .get()
+            .getState()
+            .metadata()
+            .index(secondIndex)
+            .getVersion();
+        refresh(secondIndex);
+        ensureGreen(secondIndex);
+
+        waitNoPendingTasksOnAll();
+
+        assertRemoteProperties(secondIndex);
+
+        logger.info("---> Assert metadata version is not changed");
+        assertEquals(
+            initalMetadataVersion,
+            internalCluster().client().admin().cluster().prepareState().get().getState().metadata().index(secondIndex).getVersion()
+        );
+    }
+
+    /**
+     * Scenario:
+     * Creates an index with 1 primary, 2 replicas on 2 docrep nodes. Since the replica
+     * configuration is incorrect, the index stays YELLOW.
+     * Starts 2 more remote nodes and initiates shard relocation through allocation exclusion.
+     * After shard relocation completes, shuts down the docrep nodes and asserts remote
+     * index settings are applied even when the index is in YELLOW state
+     */
+    public void testIndexSettingsUpdatedEvenForMisconfiguredReplicas() throws Exception {
+        internalCluster().startClusterManagerOnlyNode();
+
+        logger.info("---> Starting 2 docrep nodes");
+        addRemote = false;
+        List<String> docrepNodes = internalCluster().startDataOnlyNodes(2, Settings.builder().put("node.attr._type", "docrep").build());
+        internalCluster().validateClusterFormed();
+
+        logger.info("---> Creating index with 1 primary and 2 replicas");
+        String indexName = "migration-index-allocation-exclude";
+        Settings oneReplica = Settings.builder()
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 2)
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+            .build();
+        createIndexAssertHealthAndDocrepProperties(indexName, oneReplica, this::ensureYellowAndNoInitializingShards);
+
+        logger.info("---> Starting indexing in parallel");
+        AsyncIndexingService asyncIndexingService = new AsyncIndexingService(indexName);
+        asyncIndexingService.startIndexing();
+
+        logger.info("---> Starts 2 remote enabled nodes");
+        initDocRepToRemoteMigration();
+        addRemote = true;
+        internalCluster().startDataOnlyNodes(2, Settings.builder().put("node.attr._type", "remote").build());
+        internalCluster().validateClusterFormed();
+
+        logger.info("---> Excluding docrep nodes from allocation");
+        excludeNodeSet("type", "docrep");
+        waitForRelocation();
+        waitNoPendingTasksOnAll();
+        asyncIndexingService.stopIndexing();
+
+        logger.info("---> Assert cluster has turned green since more nodes are added to the cluster");
+        ensureGreen(indexName);
+
+        logger.info("---> Assert index still has dcorep settings since replica copies are still on docrep nodes");
+        assertDocrepProperties(indexName);
+
+        logger.info("---> Stopping docrep nodes");
+        for (String node : docrepNodes) {
+            internalCluster().stopRandomNode(InternalTestCluster.nameFilter(node));
+        }
+        waitNoPendingTasksOnAll();
+        ensureYellowAndNoInitializingShards(indexName);
+
+        logger.info("---> Assert remote settings are applied");
+        assertRemoteProperties(indexName);
+    }
+
+    /**
+     * Scenario:
+     * Creates an index with 1 primary, 2 replicas on 2 docrep nodes.
+     * Starts 2 more remote nodes and initiates shard relocation through allocation exclusion.
+     * After shard relocation completes, restarts the docrep node holding extra replica shard copy
+     * and asserts remote index settings are applied as soon as the docrep replica copy is unassigned
+     */
+    public void testIndexSettingsUpdatedWhenDocrepNodeIsRestarted() throws Exception {
+        internalCluster().startClusterManagerOnlyNode();
+
+        logger.info("---> Starting 2 docrep nodes");
+        addRemote = false;
+        List<String> docrepNodes = internalCluster().startDataOnlyNodes(2, Settings.builder().put("node.attr._type", "docrep").build());
+        internalCluster().validateClusterFormed();
+
+        logger.info("---> Creating index with 1 primary and 2 replicas");
+        String indexName = "migration-index-allocation-exclude";
+        Settings oneReplica = Settings.builder()
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 2)
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+            .build();
+        createIndexAssertHealthAndDocrepProperties(indexName, oneReplica, this::ensureYellowAndNoInitializingShards);
+
+        logger.info("---> Starting indexing in parallel");
+        AsyncIndexingService asyncIndexingService = new AsyncIndexingService(indexName);
+        asyncIndexingService.startIndexing();
+
+        logger.info("---> Starts 2 remote enabled nodes");
+        initDocRepToRemoteMigration();
+        addRemote = true;
+        internalCluster().startDataOnlyNodes(2, Settings.builder().put("node.attr._type", "remote").build());
+        internalCluster().validateClusterFormed();
+
+        logger.info("---> Excluding docrep nodes from allocation");
+        excludeNodeSet("type", "docrep");
+        waitForRelocation();
+        waitNoPendingTasksOnAll();
+        asyncIndexingService.stopIndexing();
+
+        logger.info("---> Assert cluster has turned green since more nodes are added to the cluster");
+        ensureGreen(indexName);
+
+        logger.info("---> Assert index still has dcorep settings since replica copies are still on docrep nodes");
+        assertDocrepProperties(indexName);
+
+        ClusterState clusterState = internalCluster().client().admin().cluster().prepareState().get().getState();
+        DiscoveryNodes nodes = clusterState.nodes();
+
+        String docrepReplicaNodeName = "";
+        for (ShardRouting shardRouting : clusterState.routingTable().index(indexName).shard(0).getShards()) {
+            if (nodes.get(shardRouting.currentNodeId()).isRemoteStoreNode() == false) {
+                docrepReplicaNodeName = nodes.get(shardRouting.currentNodeId()).getName();
+                break;
+            }
+        }
+        excludeNodeSet("type", null);
+
+        logger.info("---> Stopping docrep node holding the replica copy");
+        internalCluster().restartNode(docrepReplicaNodeName);
+        ensureStableCluster(5);
+        waitNoPendingTasksOnAll();
+
+        logger.info("---> Assert remote index settings have been applied");
+        assertRemoteProperties(indexName);
+        logger.info("---> Assert cluster is yellow since remote index settings have been applied");
+        ensureYellowAndNoInitializingShards(indexName);
+    }
+
+    /**
+     * Scenario:
+     * Creates a docrep cluster with 3 nodes and an index with 1 primary and 2 replicas.
+     * Adds 3 more remote nodes to the cluster and moves over the primary copy from docrep
+     * to remote through _cluster/reroute. Asserts that the remote store path based metadata
+     * have been applied to the index.
+     * Moves over the first replica copy and asserts that the remote store based settings has not been applied
+     * Excludes docrep nodes from allocation to force migration of the 3rd replica copy and asserts remote
+     * store settings has been applied as all shards have moved over
+     */
+    public void testRemotePathMetadataAddedWithFirstPrimaryMovingToRemote() throws Exception {
+        String indexName = "index-1";
+        internalCluster().startClusterManagerOnlyNode();
+
+        logger.info("---> Starting 3 docrep nodes");
+        internalCluster().startDataOnlyNodes(3, Settings.builder().put("node.attr._type", "docrep").build());
+        internalCluster().validateClusterFormed();
+
+        logger.info("---> Creating index with 1 primary and 2 replicas");
+        Settings oneReplica = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 2).build();
+        createIndexAndAssertDocrepProperties(indexName, oneReplica);
+
+        logger.info("---> Adding 3 remote enabled nodes");
+        initDocRepToRemoteMigration();
+        addRemote = true;
+        List<String> remoteEnabledNodes = internalCluster().startDataOnlyNodes(
+            3,
+            Settings.builder().put("node.attr._type", "remote").build()
+        );
+
+        logger.info("---> Moving primary copy to remote enabled node");
+        String primaryNodeName = primaryNodeName(indexName);
+        assertAcked(
+            client().admin()
+                .cluster()
+                .prepareReroute()
+                .add(new MoveAllocationCommand(indexName, 0, primaryNodeName, remoteEnabledNodes.get(0)))
+                .execute()
+                .actionGet()
+        );
+        waitForRelocation();
+        waitNoPendingTasksOnAll();
+
+        logger.info("---> Assert custom remote path based metadata is applied");
+        assertCustomIndexMetadata(indexName);
+
+        logger.info("---> Moving over one replica copy to remote enabled node");
+        String replicaNodeName = replicaNodeName(indexName);
+        assertAcked(
+            client().admin()
+                .cluster()
+                .prepareReroute()
+                .add(new MoveAllocationCommand(indexName, 0, replicaNodeName, remoteEnabledNodes.get(1)))
+                .execute()
+                .actionGet()
+        );
+        waitForRelocation();
+        waitNoPendingTasksOnAll();
+
+        logger.info("---> Assert index still has docrep settings");
+        assertDocrepProperties(indexName);
+
+        logger.info("---> Excluding docrep nodes from allocation");
+        excludeNodeSet("type", "docrep");
+        waitForRelocation();
+        waitNoPendingTasksOnAll();
+
+        logger.info("---> Assert index has remote store settings");
+        assertRemoteProperties(indexName);
+    }
+
+    private void createIndexAndAssertDocrepProperties(String index, Settings settings) {
+        createIndexAssertHealthAndDocrepProperties(index, settings, this::ensureGreen);
+    }
+
+    private void createIndexAssertHealthAndDocrepProperties(
+        String index,
+        Settings settings,
+        Function<String, ClusterHealthStatus> ensureState
+    ) {
+        createIndex(index, settings);
+        refresh(index);
+        ensureState.apply(index);
+        assertDocrepProperties(index);
+    }
+
+    /**
+     * Assert current index settings have:
+     * - index.remote_store.enabled == false
+     * - index.remote_store.segment.repository == null
+     * - index.remote_store.translog.repository == null
+     * - index.replication.type == DOCUMENT
+     */
+    private void assertDocrepProperties(String index) {
+        logger.info("---> Asserting docrep index settings");
+        IndexMetadata iMd = internalCluster().client().admin().cluster().prepareState().get().getState().metadata().index(index);
+        Settings settings = iMd.getSettings();
+        assertFalse(IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING.get(settings));
+        assertFalse(IndexMetadata.INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING.exists(settings));
+        assertFalse(IndexMetadata.INDEX_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING.exists(settings));
+        assertEquals(ReplicationType.DOCUMENT, IndexMetadata.INDEX_REPLICATION_TYPE_SETTING.get(settings));
+    }
+
+    /**
+     * Assert current index settings have:
+     * - index.remote_store.enabled == true
+     * - index.remote_store.segment.repository != null
+     * - index.remote_store.translog.repository != null
+     * - index.replication.type == SEGMENT
+     * Asserts index metadata customs has the <code>remote_store</code> key
+     */
+    private void assertRemoteProperties(String index) {
+        logger.info("---> Asserting remote index settings");
+        IndexMetadata iMd = internalCluster().client().admin().cluster().prepareState().get().getState().metadata().index(index);
+        Settings settings = iMd.getSettings();
+        assertTrue(IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING.get(settings));
+        assertTrue(IndexMetadata.INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING.exists(settings));
+        assertTrue(IndexMetadata.INDEX_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING.exists(settings));
+        assertEquals(ReplicationType.SEGMENT, IndexMetadata.INDEX_REPLICATION_TYPE_SETTING.get(settings));
+        assertNotNull(iMd.getCustomData(IndexMetadata.REMOTE_STORE_CUSTOM_KEY));
+    }
+
+    /**
+     * Asserts index metadata customs has the <code>remote_store</code> key
+     */
+    private void assertCustomIndexMetadata(String index) {
+        logger.info("---> Asserting custom index metadata");
+        IndexMetadata iMd = internalCluster().client().admin().cluster().prepareState().get().getState().metadata().index(index);
+        assertNotNull(iMd.getCustomData(IndexMetadata.REMOTE_STORE_CUSTOM_KEY));
+    }
+}
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemotePrimaryLocalRecoveryIT.java b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemotePrimaryLocalRecoveryIT.java
new file mode 100644
index 0000000000000..024fc68602a19
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemotePrimaryLocalRecoveryIT.java
@@ -0,0 +1,179 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.remotemigration;
+
+import org.opensearch.action.admin.indices.stats.ShardStats;
+import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.cluster.node.DiscoveryNodes;
+import org.opensearch.cluster.routing.ShardRouting;
+import org.opensearch.common.blobstore.BlobPath;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.TimeValue;
+import org.opensearch.core.util.FileSystemUtils;
+import org.opensearch.index.remote.RemoteSegmentStats;
+import org.opensearch.index.translog.RemoteTranslogStats;
+import org.opensearch.test.InternalTestCluster;
+import org.opensearch.test.OpenSearchIntegTestCase;
+
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.SEGMENTS;
+import static org.opensearch.index.remote.RemoteStoreEnums.DataType.DATA;
+import static org.opensearch.index.store.RemoteSegmentStoreDirectory.SEGMENT_NAME_UUID_SEPARATOR;
+
+@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
+public class RemotePrimaryLocalRecoveryIT extends MigrationBaseTestCase {
+    String indexName = "idx1";
+    int numOfNodes = randomIntBetween(6, 9);
+
+    /**
+     * Tests local recovery sanity in the happy path flow
+     */
+    public void testLocalRecoveryRollingRestart() throws Exception {
+        triggerRollingRestartForRemoteMigration(0);
+        internalCluster().stopAllNodes();
+    }
+
+    /**
+     * Tests local recovery sanity during remote migration with a node restart in between
+     */
+    public void testLocalRecoveryRollingRestartAndNodeFailure() throws Exception {
+        triggerRollingRestartForRemoteMigration(0);
+
+        DiscoveryNodes discoveryNodes = internalCluster().client().admin().cluster().prepareState().get().getState().getNodes();
+        DiscoveryNode nodeToRestart = (DiscoveryNode) discoveryNodes.getDataNodes().values().toArray()[randomIntBetween(0, numOfNodes - 4)];
+        internalCluster().restartNode(nodeToRestart.getName());
+
+        Map<ShardRouting, ShardStats> shardStatsMap = internalCluster().client().admin().indices().prepareStats(indexName).get().asMap();
+        for (Map.Entry<ShardRouting, ShardStats> entry : shardStatsMap.entrySet()) {
+            ShardRouting shardRouting = entry.getKey();
+            ShardStats shardStats = entry.getValue();
+            if (nodeToRestart.equals(shardRouting.currentNodeId())) {
+                RemoteSegmentStats remoteSegmentStats = shardStats.getStats().getSegments().getRemoteSegmentStats();
+                assertTrue(remoteSegmentStats.getTotalUploadTime() > 0);
+                assertTrue(remoteSegmentStats.getUploadBytesSucceeded() > 0);
+            }
+
+            assertBusy(() -> {
+                String shardPath = getShardLevelBlobPath(
+                    client(),
+                    indexName,
+                    new BlobPath(),
+                    String.valueOf(shardRouting.getId()),
+                    SEGMENTS,
+                    DATA
+                ).buildAsString();
+                Path segmentDataRepoPath = segmentRepoPath.resolve(shardPath);
+                List<String> segmentsNFilesInRepo = Arrays.stream(FileSystemUtils.files(segmentDataRepoPath))
+                    .filter(path -> path.getFileName().toString().contains("segments_"))
+                    .map(path -> path.getFileName().toString())
+                    .collect(Collectors.toList());
+                Set<String> expectedUniqueSegmentsNFiles = segmentsNFilesInRepo.stream()
+                    .map(fileName -> fileName.split(SEGMENT_NAME_UUID_SEPARATOR)[0])
+                    .collect(Collectors.toSet());
+                assertEquals(
+                    "Expected no duplicate segments_N files in remote but duplicates were found " + segmentsNFilesInRepo,
+                    expectedUniqueSegmentsNFiles.size(),
+                    segmentsNFilesInRepo.size()
+                );
+            }, 90, TimeUnit.SECONDS);
+        }
+
+        internalCluster().stopAllNodes();
+    }
+
+    /**
+     * Tests local recovery flow sanity in the happy path flow with replicas in place
+     */
+    public void testLocalRecoveryFlowWithReplicas() throws Exception {
+        triggerRollingRestartForRemoteMigration(randomIntBetween(1, 2));
+        internalCluster().stopAllNodes();
+    }
+
+    /**
+     * Helper method to run a rolling restart for migration to remote backed cluster
+     */
+    private void triggerRollingRestartForRemoteMigration(int replicaCount) throws Exception {
+        internalCluster().startClusterManagerOnlyNodes(3);
+        internalCluster().startNodes(numOfNodes - 3);
+
+        // create index
+        Settings indexSettings = Settings.builder()
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, replicaCount)
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, randomIntBetween(1, 10))
+            .build();
+        createIndex(indexName, indexSettings);
+        ensureGreen(indexName);
+        indexBulk(indexName, randomIntBetween(100, 10000));
+        refresh(indexName);
+        indexBulk(indexName, randomIntBetween(100, 10000));
+
+        initDocRepToRemoteMigration();
+
+        // rolling restart
+        final Settings remoteNodeAttributes = remoteStoreClusterSettings(
+            REPOSITORY_NAME,
+            segmentRepoPath,
+            REPOSITORY_2_NAME,
+            translogRepoPath
+        );
+        internalCluster().rollingRestart(new InternalTestCluster.RestartCallback() {
+            // Update remote attributes
+            @Override
+            public Settings onNodeStopped(String nodeName) {
+                return remoteNodeAttributes;
+            }
+        });
+        ensureStableCluster(numOfNodes);
+        ensureGreen(TimeValue.timeValueSeconds(90), indexName);
+        assertEquals(internalCluster().size(), numOfNodes);
+
+        // Assert on remote uploads
+        Map<ShardRouting, ShardStats> shardStatsMap = internalCluster().client().admin().indices().prepareStats(indexName).get().asMap();
+        DiscoveryNodes discoveryNodes = internalCluster().client().admin().cluster().prepareState().get().getState().getNodes();
+        shardStatsMap.forEach((shardRouting, shardStats) -> {
+            if (discoveryNodes.get(shardRouting.currentNodeId()).isRemoteStoreNode() && shardRouting.primary()) {
+                RemoteSegmentStats remoteSegmentStats = shardStats.getStats().getSegments().getRemoteSegmentStats();
+                assertTrue(remoteSegmentStats.getTotalUploadTime() > 0);
+                assertTrue(remoteSegmentStats.getUploadBytesSucceeded() > 0);
+            }
+        });
+
+        // Assert on new remote uploads after seeding
+        indexBulk(indexName, randomIntBetween(100, 10000));
+        refresh(indexName);
+        indexBulk(indexName, randomIntBetween(100, 10000));
+        Map<ShardRouting, ShardStats> newShardStatsMap = internalCluster().client().admin().indices().prepareStats(indexName).get().asMap();
+        newShardStatsMap.forEach((shardRouting, shardStats) -> {
+            if (discoveryNodes.get(shardRouting.currentNodeId()).isRemoteStoreNode() && shardRouting.primary()) {
+                RemoteSegmentStats prevRemoteSegmentStats = shardStatsMap.get(shardRouting)
+                    .getStats()
+                    .getSegments()
+                    .getRemoteSegmentStats();
+                RemoteSegmentStats newRemoteSegmentStats = shardStats.getStats().getSegments().getRemoteSegmentStats();
+                assertTrue(newRemoteSegmentStats.getTotalUploadTime() > prevRemoteSegmentStats.getTotalUploadTime());
+                assertTrue(newRemoteSegmentStats.getUploadBytesSucceeded() > prevRemoteSegmentStats.getUploadBytesSucceeded());
+
+                RemoteTranslogStats prevRemoteTranslogStats = shardStatsMap.get(shardRouting)
+                    .getStats()
+                    .getTranslog()
+                    .getRemoteTranslogStats();
+                RemoteTranslogStats newRemoteTranslogStats = shardStats.getStats().getTranslog().getRemoteTranslogStats();
+                assertTrue(newRemoteTranslogStats.getUploadBytesSucceeded() > prevRemoteTranslogStats.getUploadBytesSucceeded());
+            }
+        });
+    }
+}
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemotePrimaryRelocationIT.java b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemotePrimaryRelocationIT.java
index b1c429a45a1a1..293691ace2edd 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemotePrimaryRelocationIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemotePrimaryRelocationIT.java
@@ -8,9 +8,8 @@
 
 package org.opensearch.remotemigration;
 
-import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
-
 import org.opensearch.action.DocWriteResponse;
+import org.opensearch.action.admin.cluster.health.ClusterHealthRequest;
 import org.opensearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesRequest;
 import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesResponse;
@@ -18,21 +17,25 @@
 import org.opensearch.action.delete.DeleteResponse;
 import org.opensearch.action.index.IndexResponse;
 import org.opensearch.client.Client;
+import org.opensearch.client.Requests;
 import org.opensearch.cluster.routing.allocation.command.MoveAllocationCommand;
 import org.opensearch.common.Priority;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.index.query.QueryBuilders;
+import org.opensearch.indices.recovery.RecoverySettings;
 import org.opensearch.plugins.Plugin;
 import org.opensearch.test.OpenSearchIntegTestCase;
 import org.opensearch.test.hamcrest.OpenSearchAssertions;
 import org.opensearch.test.transport.MockTransportService;
 
 import java.util.Collection;
+import java.util.List;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicInteger;
 
 import static java.util.Arrays.asList;
+import static org.opensearch.node.remotestore.RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING;
 import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
 
@@ -42,7 +45,6 @@ protected int maximumNumberOfShards() {
         return 1;
     }
 
-    // ToDo : Fix me when we support migration of replicas
     protected int maximumNumberOfReplicas() {
         return 0;
     }
@@ -51,9 +53,9 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
         return asList(MockTransportService.TestPlugin.class);
     }
 
-    public void testMixedModeRelocation() throws Exception {
-        String docRepNode = internalCluster().startNode();
-        Client client = internalCluster().client(docRepNode);
+    public void testRemotePrimaryRelocation() throws Exception {
+        List<String> docRepNodes = internalCluster().startNodes(2);
+        Client client = internalCluster().client(docRepNodes.get(0));
         ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
         updateSettingsRequest.persistentSettings(Settings.builder().put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed"));
         assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
@@ -69,10 +71,13 @@ public void testMixedModeRelocation() throws Exception {
         refresh("test");
 
         // add remote node in mixed mode cluster
-        addRemote = true;
+        setAddRemote(true);
         String remoteNode = internalCluster().startNode();
         internalCluster().validateClusterFormed();
 
+        updateSettingsRequest.persistentSettings(Settings.builder().put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store"));
+        assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
+
         String remoteNode2 = internalCluster().startNode();
         internalCluster().validateClusterFormed();
 
@@ -86,8 +91,17 @@ public void testMixedModeRelocation() throws Exception {
         int finalCurrentDoc1 = currentDoc;
         waitUntil(() -> numAutoGenDocs.get() > finalCurrentDoc1 + 5);
 
-        logger.info("-->  relocating from {} to {} ", docRepNode, remoteNode);
-        client().admin().cluster().prepareReroute().add(new MoveAllocationCommand("test", 0, docRepNode, remoteNode)).execute().actionGet();
+        // Change direction to remote store
+        updateSettingsRequest.persistentSettings(Settings.builder().put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store"));
+        assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
+
+        logger.info("-->  relocating from {} to {} ", docRepNodes, remoteNode);
+        client().admin()
+            .cluster()
+            .prepareReroute()
+            .add(new MoveAllocationCommand("test", 0, primaryNodeName("test"), remoteNode))
+            .execute()
+            .actionGet();
         ClusterHealthResponse clusterHealthResponse = client().admin()
             .cluster()
             .prepareHealth()
@@ -158,16 +172,20 @@ public void testMixedModeRelocation_RemoteSeedingFail() throws Exception {
         refresh("test");
 
         // add remote node in mixed mode cluster
-        addRemote = true;
+        setAddRemote(true);
         String remoteNode = internalCluster().startNode();
         internalCluster().validateClusterFormed();
 
-        // assert repo gets registered
-        GetRepositoriesRequest gr = new GetRepositoriesRequest(new String[] { REPOSITORY_NAME });
-        GetRepositoriesResponse getRepositoriesResponse = client.admin().cluster().getRepositories(gr).actionGet();
-        assertEquals(1, getRepositoriesResponse.repositories().size());
-
         setFailRate(REPOSITORY_NAME, 100);
+        client().admin()
+            .cluster()
+            .prepareUpdateSettings()
+            .setTransientSettings(Settings.builder().put(RecoverySettings.INDICES_INTERNAL_REMOTE_UPLOAD_TIMEOUT.getKey(), "10s"))
+            .get();
+
+        // Change direction to remote store
+        updateSettingsRequest.persistentSettings(Settings.builder().put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store"));
+        assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
 
         logger.info("--> relocating from {} to {} ", docRepNode, remoteNode);
         client().admin().cluster().prepareReroute().add(new MoveAllocationCommand("test", 0, docRepNode, remoteNode)).execute().actionGet();
@@ -181,29 +199,23 @@ public void testMixedModeRelocation_RemoteSeedingFail() throws Exception {
             .actionGet();
 
         assertTrue(clusterHealthResponse.getRelocatingShards() == 1);
-        setFailRate(REPOSITORY_NAME, 0);
-        Thread.sleep(RandomNumbers.randomIntBetween(random(), 0, 2000));
-        clusterHealthResponse = client().admin()
-            .cluster()
-            .prepareHealth()
-            .setTimeout(TimeValue.timeValueSeconds(45))
-            .setWaitForEvents(Priority.LANGUID)
-            .setWaitForNoRelocatingShards(true)
-            .execute()
-            .actionGet();
-        assertTrue(clusterHealthResponse.getRelocatingShards() == 0);
-        logger.info("--> remote to remote relocation complete");
+        // waiting more than waitForRemoteStoreSync's sleep time of 30 sec to deterministically fail
+        Thread.sleep(40000);
+
+        ClusterHealthRequest healthRequest = Requests.clusterHealthRequest()
+            .waitForNoRelocatingShards(true)
+            .waitForNoInitializingShards(true);
+        ClusterHealthResponse actionGet = client().admin().cluster().health(healthRequest).actionGet();
+        assertEquals(actionGet.getRelocatingShards(), 0);
+        assertEquals(docRepNode, primaryNodeName("test"));
+
         finished.set(true);
         indexingThread.join();
-        refresh("test");
-        OpenSearchAssertions.assertHitCount(client().prepareSearch("test").setTrackTotalHits(true).get(), numAutoGenDocs.get());
-        OpenSearchAssertions.assertHitCount(
-            client().prepareSearch("test")
-                .setTrackTotalHits(true)// extra paranoia ;)
-                .setQuery(QueryBuilders.termQuery("auto", true))
-                .get(),
-            numAutoGenDocs.get()
-        );
+        client().admin()
+            .cluster()
+            .prepareUpdateSettings()
+            .setTransientSettings(Settings.builder().put(RecoverySettings.INDICES_INTERNAL_REMOTE_UPLOAD_TIMEOUT.getKey(), (String) null))
+            .get();
     }
 
     private static Thread getIndexingThread(AtomicBoolean finished, AtomicInteger numAutoGenDocs) {
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteReplicaRecoveryIT.java b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteReplicaRecoveryIT.java
new file mode 100644
index 0000000000000..196ecb991bbc0
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteReplicaRecoveryIT.java
@@ -0,0 +1,180 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.remotemigration;
+
+import com.carrotsearch.randomizedtesting.generators.RandomNumbers;
+
+import org.opensearch.action.DocWriteResponse;
+import org.opensearch.action.admin.cluster.health.ClusterHealthResponse;
+import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
+import org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequest;
+import org.opensearch.action.delete.DeleteResponse;
+import org.opensearch.action.index.IndexResponse;
+import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.cluster.routing.allocation.command.MoveAllocationCommand;
+import org.opensearch.common.Priority;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.TimeValue;
+import org.opensearch.index.query.QueryBuilders;
+import org.opensearch.test.OpenSearchIntegTestCase;
+import org.opensearch.test.hamcrest.OpenSearchAssertions;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.opensearch.node.remotestore.RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING;
+import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING;
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
+
+@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false)
+
+public class RemoteReplicaRecoveryIT extends MigrationBaseTestCase {
+
+    protected int maximumNumberOfShards() {
+        return 1;
+    }
+
+    protected int maximumNumberOfReplicas() {
+        return 1;
+    }
+
+    protected int minimumNumberOfReplicas() {
+        return 1;
+    }
+
+    /*
+    Brings up new replica copies on remote and docrep nodes, when primary is on a remote node
+    Live indexing is happening meanwhile
+    */
+    public void testReplicaRecovery() throws Exception {
+        internalCluster().setBootstrapClusterManagerNodeIndex(0);
+        String primaryNode = internalCluster().startNode();
+        ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
+        updateSettingsRequest.persistentSettings(Settings.builder().put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed"));
+        assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
+
+        // create shard with 0 replica and 1 shard
+        client().admin().indices().prepareCreate("test").setSettings(indexSettings()).setMapping("field", "type=text").get();
+        String replicaNode = internalCluster().startNode();
+        ensureGreen("test");
+
+        AtomicInteger numAutoGenDocs = new AtomicInteger();
+        final AtomicBoolean finished = new AtomicBoolean(false);
+        Thread indexingThread = getThread(finished, numAutoGenDocs);
+
+        refresh("test");
+
+        // add remote node in mixed mode cluster
+        setAddRemote(true);
+        String remoteNode = internalCluster().startNode();
+        internalCluster().validateClusterFormed();
+
+        updateSettingsRequest.persistentSettings(Settings.builder().put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store"));
+        assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
+
+        String remoteNode2 = internalCluster().startNode();
+        internalCluster().validateClusterFormed();
+
+        // identify the primary
+
+        Thread.sleep(RandomNumbers.randomIntBetween(random(), 0, 2000));
+        logger.info("-->  relocating primary from {} to {} ", primaryNode, remoteNode);
+        client().admin()
+            .cluster()
+            .prepareReroute()
+            .add(new MoveAllocationCommand("test", 0, primaryNode, remoteNode))
+            .execute()
+            .actionGet();
+        ClusterHealthResponse clusterHealthResponse = client().admin()
+            .cluster()
+            .prepareHealth()
+            .setTimeout(TimeValue.timeValueSeconds(60))
+            .setWaitForEvents(Priority.LANGUID)
+            .setWaitForNoRelocatingShards(true)
+            .execute()
+            .actionGet();
+
+        assertEquals(0, clusterHealthResponse.getRelocatingShards());
+        logger.info("-->  relocation of primary from docrep to remote  complete");
+        Thread.sleep(RandomNumbers.randomIntBetween(random(), 0, 2000));
+
+        logger.info("--> getting up the new replicas now to doc rep node as well as remote node ");
+        // Increase replica count to 3
+        client().admin()
+            .indices()
+            .updateSettings(
+                new UpdateSettingsRequest("test").settings(
+                    Settings.builder()
+                        .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 3)
+                        .put("index.routing.allocation.exclude._name", remoteNode)
+                        .build()
+                )
+            )
+            .get();
+
+        client().admin()
+            .cluster()
+            .prepareHealth()
+            .setTimeout(TimeValue.timeValueSeconds(60))
+            .setWaitForEvents(Priority.LANGUID)
+            .setWaitForGreenStatus()
+            .execute()
+            .actionGet();
+        logger.info("-->  replica  is up now on another docrep now as well as remote node");
+
+        assertEquals(0, clusterHealthResponse.getRelocatingShards());
+
+        Thread.sleep(RandomNumbers.randomIntBetween(random(), 0, 2000));
+
+        // Stop replicas on docrep now.
+        // ToDo : Remove once we have dual replication enabled
+        client().admin()
+            .indices()
+            .updateSettings(
+                new UpdateSettingsRequest("test").settings(
+                    Settings.builder()
+                        .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)
+                        .put("index.routing.allocation.exclude._name", primaryNode + "," + replicaNode)
+                        .build()
+                )
+            )
+            .get();
+
+        finished.set(true);
+        indexingThread.join();
+        refresh("test");
+        OpenSearchAssertions.assertHitCount(client().prepareSearch("test").setTrackTotalHits(true).get(), numAutoGenDocs.get());
+        OpenSearchAssertions.assertHitCount(
+            client().prepareSearch("test")
+                .setTrackTotalHits(true)// extra paranoia ;)
+                .setQuery(QueryBuilders.termQuery("auto", true))
+                // .setPreference("_prefer_nodes:" + (remoteNode+ "," + remoteNode2))
+                .get(),
+            numAutoGenDocs.get()
+        );
+
+    }
+
+    private Thread getThread(AtomicBoolean finished, AtomicInteger numAutoGenDocs) {
+        Thread indexingThread = new Thread(() -> {
+            while (finished.get() == false && numAutoGenDocs.get() < 100) {
+                IndexResponse indexResponse = client().prepareIndex("test").setId("id").setSource("field", "value").get();
+                assertEquals(DocWriteResponse.Result.CREATED, indexResponse.getResult());
+                DeleteResponse deleteResponse = client().prepareDelete("test", "id").get();
+                assertEquals(DocWriteResponse.Result.DELETED, deleteResponse.getResult());
+                client().prepareIndex("test").setSource("auto", true).get();
+                numAutoGenDocs.incrementAndGet();
+                logger.info("Indexed {} docs here", numAutoGenDocs.get());
+            }
+        });
+        indexingThread.start();
+        return indexingThread;
+    }
+
+}
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationSettingsUpdateIT.java b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationSettingsUpdateIT.java
new file mode 100644
index 0000000000000..377bd9529ca7a
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationSettingsUpdateIT.java
@@ -0,0 +1,177 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.remotemigration;
+
+import org.opensearch.client.Client;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.settings.SettingsException;
+import org.opensearch.test.InternalTestCluster;
+import org.opensearch.test.OpenSearchIntegTestCase;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.Optional;
+
+import static org.opensearch.node.remotestore.RemoteStoreNodeService.CompatibilityMode.MIXED;
+import static org.opensearch.node.remotestore.RemoteStoreNodeService.CompatibilityMode.STRICT;
+import static org.opensearch.node.remotestore.RemoteStoreNodeService.Direction.REMOTE_STORE;
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
+
+@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
+public class RemoteStoreMigrationSettingsUpdateIT extends RemoteStoreMigrationShardAllocationBaseTestCase {
+
+    private Client client;
+    private String nonRemoteNodeName;
+    private String remoteNodeName;
+
+    // remote store backed index setting tests
+
+    public void testNewIndexIsRemoteStoreBackedForRemoteStoreDirectionAndMixedMode() {
+        logger.info("Initialize cluster: gives non remote cluster manager");
+        initializeCluster(false);
+
+        String indexName1 = "test_index_1";
+        String indexName2 = "test_index_2";
+
+        logger.info("Add non-remote node");
+        addRemote = false;
+        String nonRemoteNodeName = internalCluster().startNode();
+        internalCluster().validateClusterFormed();
+        assertNodeInCluster(nonRemoteNodeName);
+
+        logger.info("Create an index");
+        prepareIndexWithoutReplica(Optional.of(indexName1));
+
+        logger.info("Verify that non remote-backed index is created");
+        assertNonRemoteStoreBackedIndex(indexName1);
+
+        logger.info("Set mixed cluster compatibility mode and remote_store direction");
+        setClusterMode(MIXED.mode);
+        setDirection(REMOTE_STORE.direction);
+
+        logger.info("Add remote node");
+        addRemote = true;
+        String remoteNodeName = internalCluster().startNode();
+        internalCluster().validateClusterFormed();
+        assertNodeInCluster(remoteNodeName);
+
+        logger.info("Create another index");
+        prepareIndexWithoutReplica(Optional.of(indexName2));
+
+        logger.info("Verify that remote backed index is created");
+        assertRemoteStoreBackedIndex(indexName2);
+    }
+
+    public void testNewRestoredIndexIsRemoteStoreBackedForRemoteStoreDirectionAndMixedMode() throws Exception {
+        logger.info("Initialize cluster: gives non remote cluster manager");
+        initializeCluster(false);
+
+        logger.info("Add remote and non-remote nodes");
+        setClusterMode(MIXED.mode);
+        addRemote = false;
+        String nonRemoteNodeName = internalCluster().startNode();
+        addRemote = true;
+        String remoteNodeName = internalCluster().startNode();
+        internalCluster().validateClusterFormed();
+        assertNodeInCluster(nonRemoteNodeName);
+        assertNodeInCluster(remoteNodeName);
+
+        logger.info("Create a non remote-backed index");
+        createIndex(TEST_INDEX, 0);
+
+        logger.info("Verify that non remote stored backed index is created");
+        assertNonRemoteStoreBackedIndex(TEST_INDEX);
+
+        logger.info("Create repository");
+        String snapshotName = "test-snapshot";
+        String snapshotRepoName = "test-restore-snapshot-repo";
+        Path snapshotRepoNameAbsolutePath = randomRepoPath().toAbsolutePath();
+        assertAcked(
+            clusterAdmin().preparePutRepository(snapshotRepoName)
+                .setType("fs")
+                .setSettings(Settings.builder().put("location", snapshotRepoNameAbsolutePath))
+        );
+
+        logger.info("Create snapshot of non remote stored backed index");
+
+        createSnapshot(snapshotRepoName, snapshotName, TEST_INDEX);
+
+        logger.info("Restore index from snapshot under NONE direction");
+        String restoredIndexName1 = TEST_INDEX + "-restored1";
+        restoreSnapshot(snapshotRepoName, snapshotName, restoredIndexName1);
+        ensureGreen(restoredIndexName1);
+
+        logger.info("Verify that restored index is non remote-backed");
+        assertNonRemoteStoreBackedIndex(restoredIndexName1);
+
+        logger.info("Restore index from snapshot under REMOTE_STORE direction");
+        setDirection(REMOTE_STORE.direction);
+        String restoredIndexName2 = TEST_INDEX + "-restored2";
+        restoreSnapshot(snapshotRepoName, snapshotName, restoredIndexName2);
+        ensureGreen(restoredIndexName2);
+
+        logger.info("Verify that restored index is non remote-backed");
+        assertRemoteStoreBackedIndex(restoredIndexName2);
+    }
+
+    // compatibility mode setting test
+
+    public void testSwitchToStrictMode() throws Exception {
+        createMixedModeCluster();
+
+        logger.info("Attempt switching to strict mode");
+        SettingsException exception = assertThrows(SettingsException.class, () -> setClusterMode(STRICT.mode));
+        assertEquals(
+            "can not switch to STRICT compatibility mode when the cluster contains both remote and non-remote nodes",
+            exception.getMessage()
+        );
+
+        stopRemoteNode();
+
+        logger.info("Attempt switching to strict mode");
+        setClusterMode(STRICT.mode);
+    }
+
+    public void testClearCompatibilityModeSetting() throws Exception {
+        createMixedModeCluster();
+        stopRemoteNode();
+
+        logger.info("Attempt clearing compatibility mode");
+        clearClusterMode();
+    }
+
+    private void stopRemoteNode() throws IOException {
+        logger.info("Stop remote node so that cluster had only non-remote nodes");
+        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(remoteNodeName));
+        ensureStableCluster(2);
+    }
+
+    private void createMixedModeCluster() {
+        logger.info("Initialize cluster");
+        initializeCluster(false);
+
+        logger.info("Create a mixed mode cluster");
+        setClusterMode(MIXED.mode);
+        addRemote = true;
+        remoteNodeName = internalCluster().startNode();
+        addRemote = false;
+        nonRemoteNodeName = internalCluster().startNode();
+        internalCluster().validateClusterFormed();
+        assertNodeInCluster(remoteNodeName);
+        assertNodeInCluster(nonRemoteNodeName);
+    }
+
+    // bootstrap a cluster
+    private void initializeCluster(boolean remoteClusterManager) {
+        addRemote = remoteClusterManager;
+        internalCluster().startClusterManagerOnlyNode();
+        client = internalCluster().client();
+    }
+
+}
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationShardAllocationBaseTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationShardAllocationBaseTestCase.java
new file mode 100644
index 0000000000000..3591846a93bc8
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationShardAllocationBaseTestCase.java
@@ -0,0 +1,363 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.remotemigration;
+
+import org.opensearch.action.admin.cluster.allocation.ClusterAllocationExplanation;
+import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
+import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
+import org.opensearch.action.support.ActiveShardCount;
+import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.cluster.node.DiscoveryNodes;
+import org.opensearch.cluster.routing.IndexShardRoutingTable;
+import org.opensearch.cluster.routing.ShardRouting;
+import org.opensearch.cluster.routing.ShardRoutingState;
+import org.opensearch.cluster.routing.allocation.AllocateUnassignedDecision;
+import org.opensearch.cluster.routing.allocation.MoveDecision;
+import org.opensearch.cluster.routing.allocation.NodeAllocationResult;
+import org.opensearch.cluster.routing.allocation.decider.Decision;
+import org.opensearch.common.Nullable;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.core.rest.RestStatus;
+import org.opensearch.index.IndexSettings;
+import org.opensearch.indices.replication.common.ReplicationType;
+import org.opensearch.snapshots.SnapshotInfo;
+import org.opensearch.snapshots.SnapshotState;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY;
+import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED;
+import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY;
+import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE;
+import static org.opensearch.index.IndexSettings.INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING;
+import static org.opensearch.node.remotestore.RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING;
+import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING;
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
+
+public class RemoteStoreMigrationShardAllocationBaseTestCase extends MigrationBaseTestCase {
+    protected static final String TEST_INDEX = "test_index";
+    protected static final String NAME = "remote_store_migration";
+
+    protected final ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
+
+    // set the compatibility mode of cluster [strict, mixed]
+    protected void setClusterMode(String mode) {
+        updateSettingsRequest.persistentSettings(Settings.builder().put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), mode));
+        assertAcked(internalCluster().client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
+    }
+
+    protected void clearClusterMode() {
+        updateSettingsRequest.persistentSettings(Settings.builder().putNull(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey()));
+        assertAcked(internalCluster().client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
+    }
+
+    // set the migration direction for cluster [remote_store, docrep, none]
+    protected void setDirection(String direction) {
+        updateSettingsRequest.persistentSettings(Settings.builder().put(MIGRATION_DIRECTION_SETTING.getKey(), direction));
+        assertAcked(internalCluster().client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
+    }
+
+    // verify that the given nodeName exists in cluster
+    protected DiscoveryNode assertNodeInCluster(String nodeName) {
+        Map<String, DiscoveryNode> nodes = internalCluster().client().admin().cluster().prepareState().get().getState().nodes().getNodes();
+        DiscoveryNode discoveryNode = null;
+        for (Map.Entry<String, DiscoveryNode> entry : nodes.entrySet()) {
+            DiscoveryNode node = entry.getValue();
+            if (node.getName().equals(nodeName)) {
+                discoveryNode = node;
+                break;
+            }
+        }
+        assertNotNull(discoveryNode);
+        return discoveryNode;
+    }
+
+    // returns a comma-separated list of node names excluding `except`
+    protected String allNodesExcept(String except) {
+        StringBuilder exclude = new StringBuilder();
+        DiscoveryNodes allNodes = internalCluster().client().admin().cluster().prepareState().get().getState().nodes();
+        for (DiscoveryNode node : allNodes) {
+            if (node.getName().equals(except) == false) {
+                exclude.append(node.getName()).append(",");
+            }
+        }
+        return exclude.toString();
+    }
+
+    // create a new test index with un-allocated primary and no replicas
+    protected void prepareIndexWithoutReplica(Optional<String> name) {
+        String indexName = name.orElse(TEST_INDEX);
+        internalCluster().client()
+            .admin()
+            .indices()
+            .prepareCreate(indexName)
+            .setSettings(
+                Settings.builder()
+                    .put("index.number_of_shards", 1)
+                    .put("index.number_of_replicas", 0)
+                    .put("index.routing.allocation.exclude._name", allNodesExcept(null))
+            )
+            .execute()
+            .actionGet();
+    }
+
+    // create a new test index with allocated primary and 1 unallocated replica
+    public void prepareIndexWithAllocatedPrimary(DiscoveryNode primaryShardNode, Optional<String> name) {
+        String indexName = name.orElse(TEST_INDEX);
+        internalCluster().client()
+            .admin()
+            .indices()
+            .prepareCreate(indexName)
+            .setSettings(
+                Settings.builder()
+                    .put("index.number_of_shards", 1)
+                    .put("index.number_of_replicas", 1)
+                    .put("index.routing.allocation.include._name", primaryShardNode.getName())
+                    .put("index.routing.allocation.exclude._name", allNodesExcept(primaryShardNode.getName()))
+            )
+            .setWaitForActiveShards(ActiveShardCount.ONE)
+            .execute()
+            .actionGet();
+
+        ensureYellowAndNoInitializingShards(TEST_INDEX);
+
+        logger.info(" --> verify allocation of primary shard");
+        assertAllocation(true, primaryShardNode);
+
+        logger.info(" --> verify non-allocation of replica shard");
+        assertNonAllocation(false);
+    }
+
+    protected ShardRouting getShardRouting(boolean isPrimary) {
+        IndexShardRoutingTable table = internalCluster().client()
+            .admin()
+            .cluster()
+            .prepareState()
+            .execute()
+            .actionGet()
+            .getState()
+            .getRoutingTable()
+            .index(TEST_INDEX)
+            .shard(0);
+        return (isPrimary ? table.primaryShard() : table.replicaShards().get(0));
+    }
+
+    // obtain decision for allocation/relocation of a shard to a given node
+    protected Decision getDecisionForTargetNode(
+        DiscoveryNode targetNode,
+        boolean isPrimary,
+        boolean includeYesDecisions,
+        boolean isRelocation
+    ) {
+        ClusterAllocationExplanation explanation = internalCluster().client()
+            .admin()
+            .cluster()
+            .prepareAllocationExplain()
+            .setIndex(TEST_INDEX)
+            .setShard(0)
+            .setPrimary(isPrimary)
+            .setIncludeYesDecisions(includeYesDecisions)
+            .get()
+            .getExplanation();
+
+        Decision requiredDecision = null;
+        List<NodeAllocationResult> nodeAllocationResults;
+        if (isRelocation) {
+            MoveDecision moveDecision = explanation.getShardAllocationDecision().getMoveDecision();
+            nodeAllocationResults = moveDecision.getNodeDecisions();
+        } else {
+            AllocateUnassignedDecision allocateUnassignedDecision = explanation.getShardAllocationDecision().getAllocateDecision();
+            nodeAllocationResults = allocateUnassignedDecision.getNodeDecisions();
+        }
+
+        for (NodeAllocationResult nodeAllocationResult : nodeAllocationResults) {
+            if (nodeAllocationResult.getNode().equals(targetNode)) {
+                for (Decision decision : nodeAllocationResult.getCanAllocateDecision().getDecisions()) {
+                    if (decision.label().equals(NAME)) {
+                        requiredDecision = decision;
+                        break;
+                    }
+                }
+            }
+        }
+
+        assertNotNull(requiredDecision);
+        return requiredDecision;
+    }
+
+    // get allocation and relocation decisions for all nodes
+    protected void excludeAllNodes() {
+        assertAcked(
+            internalCluster().client()
+                .admin()
+                .indices()
+                .prepareUpdateSettings(TEST_INDEX)
+                .setSettings(
+                    Settings.builder()
+                        .put("index.routing.allocation.include._name", "")
+                        .put("index.routing.allocation.exclude._name", allNodesExcept(null))
+                )
+                .execute()
+                .actionGet()
+        );
+    }
+
+    protected void includeAllNodes() {
+        assertAcked(
+            internalCluster().client()
+                .admin()
+                .indices()
+                .prepareUpdateSettings(TEST_INDEX)
+                .setSettings(
+                    Settings.builder()
+                        .put("index.routing.allocation.exclude._name", "")
+                        .put("index.routing.allocation.include._name", allNodesExcept(null))
+                )
+                .execute()
+                .actionGet()
+        );
+    }
+
+    protected void attemptAllocation(@Nullable String targetNodeName) {
+        Settings.Builder settingsBuilder;
+        if (targetNodeName != null) {
+            settingsBuilder = Settings.builder()
+                .put("index.routing.allocation.include._name", targetNodeName)
+                .put("index.routing.allocation.exclude._name", allNodesExcept(targetNodeName));
+        } else {
+            String clusterManagerNodeName = internalCluster().client()
+                .admin()
+                .cluster()
+                .prepareState()
+                .execute()
+                .actionGet()
+                .getState()
+                .getNodes()
+                .getClusterManagerNode()
+                .getName();
+            // to allocate freely among all nodes other than cluster-manager node
+            settingsBuilder = Settings.builder()
+                .put("index.routing.allocation.include._name", allNodesExcept(clusterManagerNodeName))
+                .put("index.routing.allocation.exclude._name", clusterManagerNodeName);
+        }
+        internalCluster().client().admin().indices().prepareUpdateSettings(TEST_INDEX).setSettings(settingsBuilder).execute().actionGet();
+    }
+
+    // verify that shard does not exist at targetNode
+    protected void assertNonAllocation(boolean isPrimary) {
+        if (isPrimary) {
+            ensureRed(TEST_INDEX);
+        } else {
+            ensureYellowAndNoInitializingShards(TEST_INDEX);
+        }
+        ShardRouting shardRouting = getShardRouting(isPrimary);
+        assertFalse(shardRouting.active());
+        assertNull(shardRouting.currentNodeId());
+        assertEquals(ShardRoutingState.UNASSIGNED, shardRouting.state());
+    }
+
+    // verify that shard exists at targetNode
+    protected void assertAllocation(boolean isPrimary, @Nullable DiscoveryNode targetNode) {
+        ShardRouting shardRouting = getShardRouting(isPrimary);
+        assertTrue(shardRouting.active());
+        assertNotNull(shardRouting.currentNodeId());
+        if (targetNode != null) {
+            assertEquals(targetNode.getId(), shardRouting.currentNodeId());
+        }
+    }
+
+    // create a snapshot
+    public static SnapshotInfo createSnapshot(String snapshotRepoName, String snapshotName, String... indices) {
+        SnapshotInfo snapshotInfo = internalCluster().client()
+            .admin()
+            .cluster()
+            .prepareCreateSnapshot(snapshotRepoName, snapshotName)
+            .setIndices(indices)
+            .setWaitForCompletion(true)
+            .get()
+            .getSnapshotInfo();
+
+        assertEquals(SnapshotState.SUCCESS, snapshotInfo.state());
+        assertTrue(snapshotInfo.successfulShards() > 0);
+        assertEquals(0, snapshotInfo.failedShards());
+        return snapshotInfo;
+    }
+
+    // create new index
+    public static void createIndex(String indexName, int replicaCount) {
+        assertAcked(
+            internalCluster().client()
+                .admin()
+                .indices()
+                .prepareCreate(indexName)
+                .setSettings(
+                    Settings.builder()
+                        .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                        .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, replicaCount)
+                        .build()
+                )
+                .get()
+        );
+    }
+
+    // restore indices from a snapshot
+    public static RestoreSnapshotResponse restoreSnapshot(String snapshotRepoName, String snapshotName, String restoredIndexName) {
+        RestoreSnapshotResponse restoreSnapshotResponse = internalCluster().client()
+            .admin()
+            .cluster()
+            .prepareRestoreSnapshot(snapshotRepoName, snapshotName)
+            .setWaitForCompletion(false)
+            .setIndices(TEST_INDEX)
+            .setRenamePattern(TEST_INDEX)
+            .setRenameReplacement(restoredIndexName)
+            .get();
+        assertEquals(restoreSnapshotResponse.status(), RestStatus.ACCEPTED);
+        return restoreSnapshotResponse;
+    }
+
+    // verify that the created index is not remote store backed
+    public static void assertNonRemoteStoreBackedIndex(String indexName) {
+        Settings indexSettings = internalCluster().client()
+            .admin()
+            .indices()
+            .prepareGetIndex()
+            .execute()
+            .actionGet()
+            .getSettings()
+            .get(indexName);
+        assertEquals(ReplicationType.DOCUMENT.toString(), indexSettings.get(SETTING_REPLICATION_TYPE));
+        assertNull(indexSettings.get(SETTING_REMOTE_STORE_ENABLED));
+        assertNull(indexSettings.get(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY));
+        assertNull(indexSettings.get(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY));
+    }
+
+    // verify that the created index is remote store backed
+    public static void assertRemoteStoreBackedIndex(String indexName) {
+        Settings indexSettings = internalCluster().client()
+            .admin()
+            .indices()
+            .prepareGetIndex()
+            .execute()
+            .actionGet()
+            .getSettings()
+            .get(indexName);
+        assertEquals(ReplicationType.SEGMENT.toString(), indexSettings.get(SETTING_REPLICATION_TYPE));
+        assertEquals("true", indexSettings.get(SETTING_REMOTE_STORE_ENABLED));
+        assertEquals(REPOSITORY_NAME, indexSettings.get(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY));
+        assertEquals(REPOSITORY_2_NAME, indexSettings.get(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY));
+        assertEquals(
+            IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL,
+            INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.get(indexSettings)
+        );
+    }
+
+}
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationTestCase.java
index a31d203058565..4b1c91f1d57ca 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationTestCase.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotemigration/RemoteStoreMigrationTestCase.java
@@ -8,32 +8,48 @@
 
 package org.opensearch.remotemigration;
 
+import org.opensearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesRequest;
 import org.opensearch.action.admin.cluster.repositories.get.GetRepositoriesResponse;
 import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
 import org.opensearch.client.Client;
+import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.common.Priority;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.TimeValue;
+import org.opensearch.index.query.QueryBuilders;
+import org.opensearch.repositories.blobstore.BlobStoreRepository;
+import org.opensearch.snapshots.SnapshotInfo;
 import org.opensearch.test.OpenSearchIntegTestCase;
+import org.opensearch.test.hamcrest.OpenSearchAssertions;
 
+import java.nio.file.Path;
 import java.util.List;
+import java.util.Map;
 
 import static org.opensearch.node.remotestore.RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING;
 import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING;
-import static org.opensearch.remotestore.RemoteStoreBaseIntegTestCase.remoteStoreClusterSettings;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
+import static org.hamcrest.Matchers.equalTo;
 
 @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0, autoManageMasterNodes = false)
 public class RemoteStoreMigrationTestCase extends MigrationBaseTestCase {
+    protected int maximumNumberOfReplicas() {
+        return 1;
+    }
+
+    protected int minimumNumberOfReplicas() {
+        return 1;
+    }
+
     public void testMixedModeAddRemoteNodes() throws Exception {
         internalCluster().setBootstrapClusterManagerNodeIndex(0);
         List<String> cmNodes = internalCluster().startNodes(1);
         Client client = internalCluster().client(cmNodes.get(0));
-        ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
-        updateSettingsRequest.persistentSettings(Settings.builder().put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed"));
-        assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
+        initDocRepToRemoteMigration();
 
         // add remote node in mixed mode cluster
-        addRemote = true;
+        setAddRemote(true);
         internalCluster().startNode();
         internalCluster().startNode();
         internalCluster().validateClusterFormed();
@@ -44,7 +60,7 @@ public void testMixedModeAddRemoteNodes() throws Exception {
         assertEquals(1, getRepositoriesResponse.repositories().size());
 
         // add docrep mode in mixed mode cluster
-        addRemote = true;
+        setAddRemote(true);
         internalCluster().startNode();
         assertBusy(() -> {
             assertEquals(client.admin().cluster().prepareClusterStats().get().getNodes().size(), internalCluster().getNodeNames().length);
@@ -71,4 +87,129 @@ public void testMigrationDirections() {
         updateSettingsRequest.persistentSettings(Settings.builder().put(MIGRATION_DIRECTION_SETTING.getKey(), "random"));
         assertThrows(IllegalArgumentException.class, () -> client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
     }
+
+    public void testNoShallowSnapshotInMixedMode() throws Exception {
+        logger.info("Initialize remote cluster");
+        addRemote = true;
+        internalCluster().setBootstrapClusterManagerNodeIndex(0);
+        List<String> cmNodes = internalCluster().startNodes(1);
+        Client client = internalCluster().client(cmNodes.get(0));
+
+        logger.info("Add remote node");
+        internalCluster().startNode();
+        internalCluster().validateClusterFormed();
+
+        logger.info("Create remote backed index");
+        RemoteStoreMigrationShardAllocationBaseTestCase.createIndex("test", 0);
+        RemoteStoreMigrationShardAllocationBaseTestCase.assertRemoteStoreBackedIndex("test");
+
+        logger.info("Create shallow snapshot setting enabled repo");
+        String shallowSnapshotRepoName = "shallow-snapshot-repo-name";
+        Path shallowSnapshotRepoPath = randomRepoPath();
+        assertAcked(
+            clusterAdmin().preparePutRepository(shallowSnapshotRepoName)
+                .setType("fs")
+                .setSettings(
+                    Settings.builder()
+                        .put("location", shallowSnapshotRepoPath)
+                        .put(BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY.getKey(), Boolean.TRUE)
+                )
+        );
+
+        logger.info("Verify shallow snapshot creation");
+        final String snapshot1 = "snapshot1";
+        SnapshotInfo snapshotInfo1 = RemoteStoreMigrationShardAllocationBaseTestCase.createSnapshot(
+            shallowSnapshotRepoName,
+            snapshot1,
+            "test"
+        );
+        assertEquals(snapshotInfo1.isRemoteStoreIndexShallowCopyEnabled(), true);
+
+        logger.info("Set MIXED compatibility mode");
+        ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
+        updateSettingsRequest.persistentSettings(Settings.builder().put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed"));
+        assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
+
+        logger.info("Verify that new snapshot is not shallow");
+        final String snapshot2 = "snapshot2";
+        SnapshotInfo snapshotInfo2 = RemoteStoreMigrationShardAllocationBaseTestCase.createSnapshot(shallowSnapshotRepoName, snapshot2);
+        assertEquals(snapshotInfo2.isRemoteStoreIndexShallowCopyEnabled(), false);
+    }
+
+    /*
+    Tests end to end remote migration via Blue Green mechanism
+    - Starts docrep nodes with multiple nodes, indices, replicas copies
+    - Adds remote nodes to cluster
+    - Excludes docrep nodes.
+    - Asserts all shards are migrated to remote store
+    - Asserts doc count across all shards
+    - Continuos indexing with refresh/flush happening
+     */
+    public void testEndToEndRemoteMigration() throws Exception {
+        internalCluster().setBootstrapClusterManagerNodeIndex(0);
+        List<String> docRepNodes = internalCluster().startNodes(2);
+        ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
+        updateSettingsRequest.persistentSettings(Settings.builder().put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed"));
+        assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
+        client().admin().indices().prepareCreate("test").setSettings(indexSettings()).setMapping("field", "type=text").get();
+        ensureGreen("test");
+
+        logger.info("---> Starting doc ingestion in parallel thread");
+        AsyncIndexingService asyncIndexingService = new AsyncIndexingService("test");
+        asyncIndexingService.startIndexing();
+
+        setAddRemote(true);
+
+        updateSettingsRequest.persistentSettings(
+            Settings.builder()
+                .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), "mixed")
+                .put(MIGRATION_DIRECTION_SETTING.getKey(), "remote_store")
+        );
+        assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
+
+        internalCluster().startNodes(2);
+
+        assertAcked(
+            internalCluster().client()
+                .admin()
+                .indices()
+                .prepareUpdateSettings()
+                .setIndices("test")
+                .setSettings(
+                    Settings.builder()
+                        .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)
+                        .put("index.routing.allocation.exclude._name", String.join(",", docRepNodes))
+                        .build()
+                )
+                .get()
+        );
+
+        ClusterHealthResponse clusterHealthResponse = client().admin()
+            .cluster()
+            .prepareHealth()
+            .setTimeout(TimeValue.timeValueSeconds(45))
+            .setWaitForEvents(Priority.LANGUID)
+            .setWaitForNoRelocatingShards(true)
+            .execute()
+            .actionGet();
+        assertTrue(clusterHealthResponse.getRelocatingShards() == 0);
+        logger.info("---> Stopping indexing thread");
+        asyncIndexingService.stopIndexing();
+        Map<String, Integer> shardCountByNodeId = getShardCountByNodeId();
+        assertThat("node0 has 0 shards", shardCountByNodeId.get(docRepNodes.get(0)), equalTo(null));
+        assertThat("node1 has 0 shards", shardCountByNodeId.get(docRepNodes.get(1)), equalTo(null));
+        refresh("test");
+        waitForReplication("test");
+        OpenSearchAssertions.assertHitCount(
+            client().prepareSearch("test").setTrackTotalHits(true).get(),
+            asyncIndexingService.getIndexedDocs()
+        );
+        OpenSearchAssertions.assertHitCount(
+            client().prepareSearch("test")
+                .setTrackTotalHits(true)// extra paranoia ;)
+                .setQuery(QueryBuilders.termQuery("auto", true))
+                .get(),
+            asyncIndexingService.getIndexedDocs()
+        );
+    }
 }
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotemigration/ResizeIndexMigrationTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotemigration/ResizeIndexMigrationTestCase.java
new file mode 100644
index 0000000000000..b817906a8f828
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/opensearch/remotemigration/ResizeIndexMigrationTestCase.java
@@ -0,0 +1,217 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.remotemigration;
+
+import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
+import org.opensearch.action.admin.indices.shrink.ResizeType;
+import org.opensearch.action.support.ActiveShardCount;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.indices.replication.common.ReplicationType;
+import org.opensearch.test.OpenSearchIntegTestCase;
+
+import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE;
+import static org.opensearch.node.remotestore.RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING;
+import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING;
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
+
+@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
+public class ResizeIndexMigrationTestCase extends MigrationBaseTestCase {
+    private static final String TEST_INDEX = "test_index";
+    private final static String REMOTE_STORE_DIRECTION = "remote_store";
+    private final static String DOC_REP_DIRECTION = "docrep";
+    private final static String MIXED_MODE = "mixed";
+
+    /*
+    * This test will verify the resize request failure, when cluster mode is mixed
+    * and index is on DocRep node, and migration to remote store is in progress.
+    * */
+    public void testFailResizeIndexWhileDocRepToRemoteStoreMigration() throws Exception {
+        setAddRemote(false);
+        // create a docrep cluster
+        internalCluster().startClusterManagerOnlyNode();
+        internalCluster().validateClusterFormed();
+
+        // add a non-remote node
+        String nonRemoteNodeName = internalCluster().startDataOnlyNode();
+        internalCluster().validateClusterFormed();
+
+        logger.info("-->Create index on non-remote node and SETTING_REMOTE_STORE_ENABLED is false. Resize should not happen");
+        Settings.Builder builder = Settings.builder().put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT);
+        internalCluster().client()
+            .admin()
+            .indices()
+            .prepareCreate(TEST_INDEX)
+            .setSettings(
+                builder.put("index.number_of_shards", 10)
+                    .put("index.number_of_replicas", 0)
+                    .put("index.routing.allocation.include._name", nonRemoteNodeName)
+            )
+            .setWaitForActiveShards(ActiveShardCount.ALL)
+            .execute()
+            .actionGet();
+
+        // set mixed mode
+        ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
+        updateSettingsRequest.persistentSettings(Settings.builder().put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), MIXED_MODE));
+        assertAcked(internalCluster().client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
+
+        // add a remote node
+        addRemote = true;
+        String remoteNodeName = internalCluster().startDataOnlyNode();
+        internalCluster().validateClusterFormed();
+
+        // set remote store migration direction
+        updateSettingsRequest.persistentSettings(Settings.builder().put(MIGRATION_DIRECTION_SETTING.getKey(), REMOTE_STORE_DIRECTION));
+        assertAcked(internalCluster().client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
+
+        ResizeType resizeType;
+        int resizeShardsNum;
+        String cause;
+        switch (randomIntBetween(0, 2)) {
+            case 0:
+                resizeType = ResizeType.SHRINK;
+                resizeShardsNum = 5;
+                cause = "shrink_index";
+                break;
+            case 1:
+                resizeType = ResizeType.SPLIT;
+                resizeShardsNum = 20;
+                cause = "split_index";
+                break;
+            default:
+                resizeType = ResizeType.CLONE;
+                resizeShardsNum = 10;
+                cause = "clone_index";
+        }
+
+        internalCluster().client()
+            .admin()
+            .indices()
+            .prepareUpdateSettings(TEST_INDEX)
+            .setSettings(Settings.builder().put("index.blocks.write", true))
+            .execute()
+            .actionGet();
+
+        ensureGreen(TEST_INDEX);
+
+        Settings.Builder resizeSettingsBuilder = Settings.builder()
+            .put("index.number_of_replicas", 0)
+            .put("index.number_of_shards", resizeShardsNum)
+            .putNull("index.blocks.write");
+
+        IllegalStateException ex = expectThrows(
+            IllegalStateException.class,
+            () -> internalCluster().client()
+                .admin()
+                .indices()
+                .prepareResizeIndex(TEST_INDEX, "first_split")
+                .setResizeType(resizeType)
+                .setSettings(resizeSettingsBuilder.build())
+                .get()
+        );
+        assertEquals(
+            ex.getMessage(),
+            "Index " + resizeType + " is not allowed as remote migration mode is mixed" + " and index is remote store disabled"
+        );
+    }
+
+    /*
+     * This test will verify the resize request failure, when cluster mode is mixed
+     * and index is on Remote Store node, and migration to DocRep node is in progress.
+     * */
+    public void testFailResizeIndexWhileRemoteStoreToDocRepMigration() throws Exception {
+        // creates a remote cluster
+        setAddRemote(true);
+        internalCluster().startClusterManagerOnlyNode();
+        internalCluster().validateClusterFormed();
+
+        // add a remote node
+        String remoteNodeName = internalCluster().startDataOnlyNode();
+        internalCluster().validateClusterFormed();
+
+        logger.info("--> Create index on remote node and SETTING_REMOTE_STORE_ENABLED is true. Resize should not happen");
+        Settings.Builder builder = Settings.builder().put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT);
+        internalCluster().client()
+            .admin()
+            .indices()
+            .prepareCreate(TEST_INDEX)
+            .setSettings(
+                builder.put("index.number_of_shards", 10)
+                    .put("index.number_of_replicas", 0)
+                    .put("index.routing.allocation.include._name", remoteNodeName)
+            )
+            .setWaitForActiveShards(ActiveShardCount.ALL)
+            .execute()
+            .actionGet();
+
+        // set mixed mode
+        ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
+        updateSettingsRequest.persistentSettings(Settings.builder().put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), MIXED_MODE));
+        assertAcked(internalCluster().client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
+
+        // add a non-remote node
+        addRemote = false;
+        String nonRemoteNodeName = internalCluster().startDataOnlyNode();
+        internalCluster().validateClusterFormed();
+
+        // set docrep migration direction
+        updateSettingsRequest.persistentSettings(Settings.builder().put(MIGRATION_DIRECTION_SETTING.getKey(), DOC_REP_DIRECTION));
+        assertAcked(internalCluster().client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
+
+        ResizeType resizeType;
+        int resizeShardsNum;
+        String cause;
+        switch (randomIntBetween(0, 2)) {
+            case 0:
+                resizeType = ResizeType.SHRINK;
+                resizeShardsNum = 5;
+                cause = "shrink_index";
+                break;
+            case 1:
+                resizeType = ResizeType.SPLIT;
+                resizeShardsNum = 20;
+                cause = "split_index";
+                break;
+            default:
+                resizeType = ResizeType.CLONE;
+                resizeShardsNum = 10;
+                cause = "clone_index";
+        }
+
+        internalCluster().client()
+            .admin()
+            .indices()
+            .prepareUpdateSettings(TEST_INDEX)
+            .setSettings(Settings.builder().put("index.blocks.write", true))
+            .execute()
+            .actionGet();
+
+        ensureGreen(TEST_INDEX);
+
+        Settings.Builder resizeSettingsBuilder = Settings.builder()
+            .put("index.number_of_replicas", 0)
+            .put("index.number_of_shards", resizeShardsNum)
+            .putNull("index.blocks.write");
+
+        IllegalStateException ex = expectThrows(
+            IllegalStateException.class,
+            () -> internalCluster().client()
+                .admin()
+                .indices()
+                .prepareResizeIndex(TEST_INDEX, "first_split")
+                .setResizeType(resizeType)
+                .setSettings(resizeSettingsBuilder.build())
+                .get()
+        );
+        assertEquals(
+            ex.getMessage(),
+            "Index " + resizeType + " is not allowed as remote migration mode is mixed" + " and index is remote store enabled"
+        );
+    }
+}
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java
index fff99e65054dc..f8e5079b01a36 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteRestoreSnapshotIT.java
@@ -12,14 +12,14 @@
 import org.opensearch.action.admin.cluster.remotestore.restore.RestoreRemoteStoreRequest;
 import org.opensearch.action.admin.cluster.snapshots.restore.RestoreSnapshotResponse;
 import org.opensearch.action.admin.indices.delete.DeleteIndexRequest;
-import org.opensearch.action.admin.indices.get.GetIndexRequest;
-import org.opensearch.action.admin.indices.get.GetIndexResponse;
 import org.opensearch.action.delete.DeleteResponse;
 import org.opensearch.action.support.PlainActionFuture;
 import org.opensearch.client.Client;
 import org.opensearch.client.Requests;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.common.Nullable;
+import org.opensearch.common.blobstore.BlobPath;
 import org.opensearch.common.io.PathUtils;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.util.io.IOUtils;
@@ -27,12 +27,14 @@
 import org.opensearch.core.rest.RestStatus;
 import org.opensearch.index.IndexService;
 import org.opensearch.index.IndexSettings;
-import org.opensearch.index.remote.RemoteStorePathType;
+import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm;
+import org.opensearch.index.remote.RemoteStoreEnums.PathType;
 import org.opensearch.index.shard.IndexShard;
 import org.opensearch.indices.IndicesService;
 import org.opensearch.indices.replication.common.ReplicationType;
 import org.opensearch.snapshots.AbstractSnapshotIntegTestCase;
 import org.opensearch.snapshots.SnapshotInfo;
+import org.opensearch.snapshots.SnapshotRestoreException;
 import org.opensearch.snapshots.SnapshotState;
 import org.opensearch.test.InternalTestCluster;
 import org.opensearch.test.OpenSearchIntegTestCase;
@@ -46,14 +48,18 @@
 import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.concurrent.ExecutionException;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
-import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY;
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED;
-import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_STORE_PATH_PREFIX_TYPE_SETTING;
+import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.SEGMENTS;
+import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.TRANSLOG;
+import static org.opensearch.index.remote.RemoteStoreEnums.DataType.DATA;
+import static org.opensearch.index.remote.RemoteStoreEnums.DataType.METADATA;
+import static org.opensearch.indices.RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
@@ -110,7 +116,7 @@ private void assertDocsPresentInIndex(Client client, String indexName, int numOf
         }
     }
 
-    public void testRestoreOperationsShallowCopyEnabled() throws IOException, ExecutionException, InterruptedException {
+    public void testRestoreOperationsShallowCopyEnabled() throws Exception {
         String clusterManagerNode = internalCluster().startClusterManagerOnlyNode();
         String primary = internalCluster().startDataOnlyNode();
         String indexName1 = "testindex1";
@@ -121,8 +127,6 @@ public void testRestoreOperationsShallowCopyEnabled() throws IOException, Execut
         Path absolutePath1 = randomRepoPath().toAbsolutePath();
         logger.info("Snapshot Path [{}]", absolutePath1);
         String restoredIndexName1 = indexName1 + "-restored";
-        String restoredIndexName1Seg = indexName1 + "-restored-seg";
-        String restoredIndexName1Doc = indexName1 + "-restored-doc";
         String restoredIndexName2 = indexName2 + "-restored";
 
         createRepository(snapshotRepoName, "fs", getRepositorySettings(absolutePath1, true));
@@ -204,60 +208,6 @@ public void testRestoreOperationsShallowCopyEnabled() throws IOException, Execut
         indexDocuments(client, restoredIndexName1, numDocsInIndex1, numDocsInIndex1 + 2);
         ensureGreen(restoredIndexName1);
         assertDocsPresentInIndex(client, restoredIndexName1, numDocsInIndex1 + 2);
-
-        // restore index as seg rep enabled with remote store and remote translog disabled
-        RestoreSnapshotResponse restoreSnapshotResponse3 = client.admin()
-            .cluster()
-            .prepareRestoreSnapshot(snapshotRepoName, snapshotName1)
-            .setWaitForCompletion(false)
-            .setIgnoreIndexSettings(IndexMetadata.SETTING_REMOTE_STORE_ENABLED)
-            .setIndices(indexName1)
-            .setRenamePattern(indexName1)
-            .setRenameReplacement(restoredIndexName1Seg)
-            .get();
-        assertEquals(restoreSnapshotResponse3.status(), RestStatus.ACCEPTED);
-        ensureGreen(restoredIndexName1Seg);
-
-        GetIndexResponse getIndexResponse = client.admin()
-            .indices()
-            .getIndex(new GetIndexRequest().indices(restoredIndexName1Seg).includeDefaults(true))
-            .get();
-        indexSettings = getIndexResponse.settings().get(restoredIndexName1Seg);
-        assertNull(indexSettings.get(SETTING_REMOTE_STORE_ENABLED));
-        assertNull(indexSettings.get(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, null));
-        assertEquals(ReplicationType.SEGMENT.toString(), indexSettings.get(IndexMetadata.SETTING_REPLICATION_TYPE));
-        assertDocsPresentInIndex(client, restoredIndexName1Seg, numDocsInIndex1);
-        // indexing some new docs and validating
-        indexDocuments(client, restoredIndexName1Seg, numDocsInIndex1, numDocsInIndex1 + 2);
-        ensureGreen(restoredIndexName1Seg);
-        assertDocsPresentInIndex(client, restoredIndexName1Seg, numDocsInIndex1 + 2);
-
-        // restore index as doc rep based from shallow copy snapshot
-        RestoreSnapshotResponse restoreSnapshotResponse4 = client.admin()
-            .cluster()
-            .prepareRestoreSnapshot(snapshotRepoName, snapshotName1)
-            .setWaitForCompletion(false)
-            .setIgnoreIndexSettings(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, IndexMetadata.SETTING_REPLICATION_TYPE)
-            .setIndices(indexName1)
-            .setRenamePattern(indexName1)
-            .setRenameReplacement(restoredIndexName1Doc)
-            .get();
-        assertEquals(restoreSnapshotResponse4.status(), RestStatus.ACCEPTED);
-        ensureGreen(restoredIndexName1Doc);
-
-        getIndexResponse = client.admin()
-            .indices()
-            .getIndex(new GetIndexRequest().indices(restoredIndexName1Doc).includeDefaults(true))
-            .get();
-        indexSettings = getIndexResponse.settings().get(restoredIndexName1Doc);
-        assertNull(indexSettings.get(SETTING_REMOTE_STORE_ENABLED));
-        assertNull(indexSettings.get(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, null));
-        assertNull(indexSettings.get(IndexMetadata.SETTING_REPLICATION_TYPE));
-        assertDocsPresentInIndex(client, restoredIndexName1Doc, numDocsInIndex1);
-        // indexing some new docs and validating
-        indexDocuments(client, restoredIndexName1Doc, numDocsInIndex1, numDocsInIndex1 + 2);
-        ensureGreen(restoredIndexName1Doc);
-        assertDocsPresentInIndex(client, restoredIndexName1Doc, numDocsInIndex1 + 2);
     }
 
     /**
@@ -276,6 +226,11 @@ public void testRemoteStoreCustomDataOnIndexCreationAndRestore() {
         String restoredIndexName1version1 = indexName1 + "-restored-1";
         String restoredIndexName1version2 = indexName1 + "-restored-2";
 
+        client(clusterManagerNode).admin()
+            .cluster()
+            .prepareUpdateSettings()
+            .setTransientSettings(Settings.builder().put(CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), PathType.FIXED))
+            .get();
         createRepository(snapshotRepoName, "fs", getRepositorySettings(absolutePath1, true));
         Client client = client();
         Settings indexSettings = getIndexSettings(1, 0).build();
@@ -283,7 +238,7 @@ public void testRemoteStoreCustomDataOnIndexCreationAndRestore() {
 
         indexDocuments(client, indexName1, randomIntBetween(5, 10));
         ensureGreen(indexName1);
-        validateRemoteStorePathType(indexName1, RemoteStorePathType.FIXED);
+        validatePathType(indexName1, PathType.FIXED);
 
         logger.info("--> snapshot");
         SnapshotInfo snapshotInfo = createSnapshot(snapshotRepoName, snapshotName1, new ArrayList<>(Arrays.asList(indexName1)));
@@ -300,14 +255,12 @@ public void testRemoteStoreCustomDataOnIndexCreationAndRestore() {
             .get();
         assertEquals(RestStatus.ACCEPTED, restoreSnapshotResponse.status());
         ensureGreen(restoredIndexName1version1);
-        validateRemoteStorePathType(restoredIndexName1version1, RemoteStorePathType.FIXED);
+        validatePathType(restoredIndexName1version1, PathType.FIXED);
 
         client(clusterManagerNode).admin()
             .cluster()
             .prepareUpdateSettings()
-            .setTransientSettings(
-                Settings.builder().put(CLUSTER_REMOTE_STORE_PATH_PREFIX_TYPE_SETTING.getKey(), RemoteStorePathType.HASHED_PREFIX)
-            )
+            .setTransientSettings(Settings.builder().put(CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), PathType.HASHED_PREFIX))
             .get();
 
         restoreSnapshotResponse = client.admin()
@@ -319,24 +272,59 @@ public void testRemoteStoreCustomDataOnIndexCreationAndRestore() {
             .get();
         assertEquals(RestStatus.ACCEPTED, restoreSnapshotResponse.status());
         ensureGreen(restoredIndexName1version2);
-        validateRemoteStorePathType(restoredIndexName1version2, RemoteStorePathType.HASHED_PREFIX);
+        validatePathType(restoredIndexName1version2, PathType.HASHED_PREFIX, PathHashAlgorithm.FNV_1A_COMPOSITE_1);
 
-        // Create index with cluster setting cluster.remote_store.index.path.prefix.type as hashed_prefix.
+        // Create index with cluster setting cluster.remote_store.index.path.type as hashed_prefix.
         indexSettings = getIndexSettings(1, 0).build();
         createIndex(indexName2, indexSettings);
         ensureGreen(indexName2);
-        validateRemoteStorePathType(indexName2, RemoteStorePathType.HASHED_PREFIX);
+        validatePathType(indexName2, PathType.HASHED_PREFIX, PathHashAlgorithm.FNV_1A_COMPOSITE_1);
 
         // Validating that custom data has not changed for indexes which were created before the cluster setting got updated
-        validateRemoteStorePathType(indexName1, RemoteStorePathType.FIXED);
+        validatePathType(indexName1, PathType.FIXED);
+
+        // Create Snapshot of index 2
+        String snapshotName2 = "test-restore-snapshot2";
+        snapshotInfo = createSnapshot(snapshotRepoName, snapshotName2, new ArrayList<>(List.of(indexName2)));
+        assertEquals(SnapshotState.SUCCESS, snapshotInfo.state());
+        assertTrue(snapshotInfo.successfulShards() > 0);
+        assertEquals(snapshotInfo.totalShards(), snapshotInfo.successfulShards());
+
+        // Update cluster settings to FIXED
+        client(clusterManagerNode).admin()
+            .cluster()
+            .prepareUpdateSettings()
+            .setTransientSettings(Settings.builder().put(CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), PathType.FIXED))
+            .get();
+
+        // Close index 2
+        assertAcked(client().admin().indices().prepareClose(indexName2));
+        restoreSnapshotResponse = client.admin()
+            .cluster()
+            .prepareRestoreSnapshot(snapshotRepoName, snapshotName2)
+            .setWaitForCompletion(false)
+            .setIndices(indexName2)
+            .get();
+        assertEquals(RestStatus.ACCEPTED, restoreSnapshotResponse.status());
+        ensureGreen(indexName2);
+
+        // Validating that custom data has not changed for testindex2 which was created before the cluster setting got updated
+        validatePathType(indexName2, PathType.HASHED_PREFIX, PathHashAlgorithm.FNV_1A_COMPOSITE_1);
+    }
+
+    private void validatePathType(String index, PathType pathType) {
+        validatePathType(index, pathType, null);
     }
 
-    private void validateRemoteStorePathType(String index, RemoteStorePathType pathType) {
+    private void validatePathType(String index, PathType pathType, @Nullable PathHashAlgorithm pathHashAlgorithm) {
         ClusterState state = client().admin().cluster().prepareState().execute().actionGet().getState();
         // Validate that the remote_store custom data is present in index metadata for the created index.
         Map<String, String> remoteCustomData = state.metadata().index(index).getCustomData(IndexMetadata.REMOTE_STORE_CUSTOM_KEY);
         assertNotNull(remoteCustomData);
-        assertEquals(pathType.toString(), remoteCustomData.get(RemoteStorePathType.NAME));
+        assertEquals(pathType.name(), remoteCustomData.get(PathType.NAME));
+        if (Objects.nonNull(pathHashAlgorithm)) {
+            assertEquals(pathHashAlgorithm.name(), remoteCustomData.get(PathHashAlgorithm.NAME));
+        }
     }
 
     public void testRestoreInSameRemoteStoreEnabledIndex() throws IOException {
@@ -440,12 +428,15 @@ public void testRestoreInSameRemoteStoreEnabledIndex() throws IOException {
     }
 
     void assertRemoteSegmentsAndTranslogUploaded(String idx) throws IOException {
-        String indexUUID = client().admin().indices().prepareGetSettings(idx).get().getSetting(idx, IndexMetadata.SETTING_INDEX_UUID);
-
-        Path remoteTranslogMetadataPath = Path.of(String.valueOf(remoteRepoPath), indexUUID, "/0/translog/metadata");
-        Path remoteTranslogDataPath = Path.of(String.valueOf(remoteRepoPath), indexUUID, "/0/translog/data");
-        Path segmentMetadataPath = Path.of(String.valueOf(remoteRepoPath), indexUUID, "/0/segments/metadata");
-        Path segmentDataPath = Path.of(String.valueOf(remoteRepoPath), indexUUID, "/0/segments/data");
+        Client client = client();
+        String path = getShardLevelBlobPath(client, idx, new BlobPath(), "0", TRANSLOG, METADATA).buildAsString();
+        Path remoteTranslogMetadataPath = Path.of(remoteRepoPath + "/" + path);
+        path = getShardLevelBlobPath(client, idx, new BlobPath(), "0", TRANSLOG, DATA).buildAsString();
+        Path remoteTranslogDataPath = Path.of(remoteRepoPath + "/" + path);
+        path = getShardLevelBlobPath(client, idx, new BlobPath(), "0", SEGMENTS, METADATA).buildAsString();
+        Path segmentMetadataPath = Path.of(remoteRepoPath + "/" + path);
+        path = getShardLevelBlobPath(client, idx, new BlobPath(), "0", SEGMENTS, DATA).buildAsString();
+        Path segmentDataPath = Path.of(remoteRepoPath + "/" + path);
 
         try (
             Stream<Path> translogMetadata = Files.list(remoteTranslogMetadataPath);
@@ -530,83 +521,6 @@ protected IndexShard getIndexShard(String node, String indexName) {
         return shardId.map(indexService::getShard).orElse(null);
     }
 
-    public void testRestoreShallowCopySnapshotWithDifferentRepo() throws IOException {
-        String clusterManagerNode = internalCluster().startClusterManagerOnlyNode();
-        String primary = internalCluster().startDataOnlyNode();
-        String indexName1 = "testindex1";
-        String indexName2 = "testindex2";
-        String snapshotRepoName = "test-restore-snapshot-repo";
-        String remoteStoreRepo2Name = "test-rs-repo-2" + TEST_REMOTE_STORE_REPO_SUFFIX;
-        String snapshotName1 = "test-restore-snapshot1";
-        Path absolutePath1 = randomRepoPath().toAbsolutePath();
-        Path absolutePath3 = randomRepoPath().toAbsolutePath();
-        String restoredIndexName1 = indexName1 + "-restored";
-
-        createRepository(snapshotRepoName, "fs", getRepositorySettings(absolutePath1, false));
-        createRepository(remoteStoreRepo2Name, "fs", absolutePath3);
-
-        Client client = client();
-        Settings indexSettings = getIndexSettings(1, 0).build();
-        createIndex(indexName1, indexSettings);
-
-        Settings indexSettings2 = getIndexSettings(1, 0).build();
-        createIndex(indexName2, indexSettings2);
-
-        final int numDocsInIndex1 = 5;
-        final int numDocsInIndex2 = 6;
-        indexDocuments(client, indexName1, numDocsInIndex1);
-        indexDocuments(client, indexName2, numDocsInIndex2);
-        ensureGreen(indexName1, indexName2);
-
-        internalCluster().startDataOnlyNode();
-
-        logger.info("--> snapshot");
-        SnapshotInfo snapshotInfo1 = createSnapshot(
-            snapshotRepoName,
-            snapshotName1,
-            new ArrayList<>(Arrays.asList(indexName1, indexName2))
-        );
-        assertThat(snapshotInfo1.successfulShards(), greaterThan(0));
-        assertThat(snapshotInfo1.successfulShards(), equalTo(snapshotInfo1.totalShards()));
-        assertThat(snapshotInfo1.state(), equalTo(SnapshotState.SUCCESS));
-
-        Settings remoteStoreIndexSettings = Settings.builder()
-            .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, remoteStoreRepo2Name)
-            .build();
-        // restore index as a remote store index with different remote store repo
-        RestoreSnapshotResponse restoreSnapshotResponse = client.admin()
-            .cluster()
-            .prepareRestoreSnapshot(snapshotRepoName, snapshotName1)
-            .setWaitForCompletion(false)
-            .setIndexSettings(remoteStoreIndexSettings)
-            .setIndices(indexName1)
-            .setRenamePattern(indexName1)
-            .setRenameReplacement(restoredIndexName1)
-            .get();
-        assertEquals(restoreSnapshotResponse.status(), RestStatus.ACCEPTED);
-        ensureGreen(restoredIndexName1);
-        assertDocsPresentInIndex(client(), restoredIndexName1, numDocsInIndex1);
-
-        // deleting data for restoredIndexName1 and restoring from remote store.
-        internalCluster().stopRandomNode(InternalTestCluster.nameFilter(primary));
-        // Re-initialize client to make sure we are not using client from stopped node.
-        client = client(clusterManagerNode);
-        assertAcked(client.admin().indices().prepareClose(restoredIndexName1));
-        client.admin()
-            .cluster()
-            .restoreRemoteStore(
-                new RestoreRemoteStoreRequest().indices(restoredIndexName1).restoreAllShards(true),
-                PlainActionFuture.newFuture()
-            );
-        ensureYellowAndNoInitializingShards(restoredIndexName1);
-        ensureGreen(restoredIndexName1);
-        // indexing some new docs and validating
-        assertDocsPresentInIndex(client, restoredIndexName1, numDocsInIndex1);
-        indexDocuments(client, restoredIndexName1, numDocsInIndex1, numDocsInIndex1 + 2);
-        ensureGreen(restoredIndexName1);
-        assertDocsPresentInIndex(client, restoredIndexName1, numDocsInIndex1 + 2);
-    }
-
     public void testRestoreShallowSnapshotRepository() throws ExecutionException, InterruptedException {
         String indexName1 = "testindex1";
         String snapshotRepoName = "test-restore-snapshot-repo";
@@ -738,4 +652,98 @@ public void testRestoreShallowSnapshotIndexAfterSnapshot() throws ExecutionExcep
         assertDocsPresentInIndex(client, restoredIndexName1, numDocsInIndex1 + 2);
     }
 
+    public void testInvalidRestoreRequestScenarios() throws Exception {
+        internalCluster().startClusterManagerOnlyNode();
+        internalCluster().startDataOnlyNode();
+        String index = "test-index";
+        String snapshotRepo = "test-restore-snapshot-repo";
+        String newRemoteStoreRepo = "test-new-rs-repo";
+        String snapshotName1 = "test-restore-snapshot1";
+        String snapshotName2 = "test-restore-snapshot2";
+        Path absolutePath1 = randomRepoPath().toAbsolutePath();
+        logger.info("Snapshot Path [{}]", absolutePath1);
+        String restoredIndex = index + "-restored";
+
+        createRepository(snapshotRepo, "fs", getRepositorySettings(absolutePath1, true));
+
+        Client client = client();
+        Settings indexSettings = getIndexSettings(1, 0).build();
+        createIndex(index, indexSettings);
+
+        final int numDocsInIndex = 5;
+        indexDocuments(client, index, numDocsInIndex);
+        ensureGreen(index);
+
+        internalCluster().startDataOnlyNode();
+        logger.info("--> snapshot");
+
+        SnapshotInfo snapshotInfo = createSnapshot(snapshotRepo, snapshotName1, new ArrayList<>(List.of(index)));
+        assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS));
+        assertThat(snapshotInfo.successfulShards(), greaterThan(0));
+        assertThat(snapshotInfo.successfulShards(), equalTo(snapshotInfo.totalShards()));
+
+        updateRepository(snapshotRepo, "fs", getRepositorySettings(absolutePath1, false));
+        SnapshotInfo snapshotInfo2 = createSnapshot(snapshotRepo, snapshotName2, new ArrayList<>(List.of(index)));
+        assertThat(snapshotInfo2.state(), equalTo(SnapshotState.SUCCESS));
+        assertThat(snapshotInfo2.successfulShards(), greaterThan(0));
+        assertThat(snapshotInfo2.successfulShards(), equalTo(snapshotInfo2.totalShards()));
+
+        DeleteResponse deleteResponse = client().prepareDelete(index, "0").execute().actionGet();
+        assertEquals(deleteResponse.getResult(), DocWriteResponse.Result.DELETED);
+        indexDocuments(client, index, numDocsInIndex, numDocsInIndex + randomIntBetween(2, 5));
+        ensureGreen(index);
+
+        // try index restore with remote store disabled
+        SnapshotRestoreException exception = expectThrows(
+            SnapshotRestoreException.class,
+            () -> client().admin()
+                .cluster()
+                .prepareRestoreSnapshot(snapshotRepo, snapshotName1)
+                .setWaitForCompletion(false)
+                .setIgnoreIndexSettings(SETTING_REMOTE_STORE_ENABLED)
+                .setIndices(index)
+                .setRenamePattern(index)
+                .setRenameReplacement(restoredIndex)
+                .get()
+        );
+        assertTrue(exception.getMessage().contains("cannot remove setting [index.remote_store.enabled] on restore"));
+
+        // try index restore with remote store repository modified
+        Settings remoteStoreIndexSettings = Settings.builder()
+            .put(IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, newRemoteStoreRepo)
+            .build();
+
+        exception = expectThrows(
+            SnapshotRestoreException.class,
+            () -> client().admin()
+                .cluster()
+                .prepareRestoreSnapshot(snapshotRepo, snapshotName1)
+                .setWaitForCompletion(false)
+                .setIndexSettings(remoteStoreIndexSettings)
+                .setIndices(index)
+                .setRenamePattern(index)
+                .setRenameReplacement(restoredIndex)
+                .get()
+        );
+        assertTrue(exception.getMessage().contains("cannot modify setting [index.remote_store.segment.repository]" + " on restore"));
+
+        // try index restore with remote store repository and translog store repository disabled
+        exception = expectThrows(
+            SnapshotRestoreException.class,
+            () -> client().admin()
+                .cluster()
+                .prepareRestoreSnapshot(snapshotRepo, snapshotName1)
+                .setWaitForCompletion(false)
+                .setIgnoreIndexSettings(
+                    IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY,
+                    IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY
+                )
+                .setIndices(index)
+                .setRenamePattern(index)
+                .setRenameReplacement(restoredIndex)
+                .get()
+        );
+        assertTrue(exception.getMessage().contains("cannot remove setting [index.remote_store.segment.repository]" + " on restore"));
+    }
+
 }
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java
index ba90cbe96e157..d7ad0daa43524 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreBaseIntegTestCase.java
@@ -28,7 +28,6 @@
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.UUIDs;
 import org.opensearch.common.settings.Settings;
-import org.opensearch.core.common.unit.ByteSizeUnit;
 import org.opensearch.core.index.Index;
 import org.opensearch.index.IndexModule;
 import org.opensearch.index.IndexService;
@@ -57,11 +56,8 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
 
-import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY;
 import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX;
 import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT;
-import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY;
-import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY;
 import static org.opensearch.repositories.fs.ReloadableFsRepository.REPOSITORIES_FAILRATE_SETTING;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
 
@@ -191,121 +187,6 @@ protected BulkResponse indexBulk(String indexName, int numDocs) {
         return client().bulk(bulkRequest).actionGet();
     }
 
-    public static Settings remoteStoreClusterSettings(String name, Path path) {
-        return remoteStoreClusterSettings(name, path, name, path);
-    }
-
-    public static Settings remoteStoreClusterSettings(
-        String segmentRepoName,
-        Path segmentRepoPath,
-        String segmentRepoType,
-        String translogRepoName,
-        Path translogRepoPath,
-        String translogRepoType
-    ) {
-        Settings.Builder settingsBuilder = Settings.builder();
-        settingsBuilder.put(
-            buildRemoteStoreNodeAttributes(
-                segmentRepoName,
-                segmentRepoPath,
-                segmentRepoType,
-                translogRepoName,
-                translogRepoPath,
-                translogRepoType,
-                false
-            )
-        );
-        return settingsBuilder.build();
-    }
-
-    public static Settings remoteStoreClusterSettings(
-        String segmentRepoName,
-        Path segmentRepoPath,
-        String translogRepoName,
-        Path translogRepoPath
-    ) {
-        Settings.Builder settingsBuilder = Settings.builder();
-        settingsBuilder.put(buildRemoteStoreNodeAttributes(segmentRepoName, segmentRepoPath, translogRepoName, translogRepoPath, false));
-        return settingsBuilder.build();
-    }
-
-    public static Settings buildRemoteStoreNodeAttributes(
-        String segmentRepoName,
-        Path segmentRepoPath,
-        String translogRepoName,
-        Path translogRepoPath,
-        boolean withRateLimiterAttributes
-    ) {
-        return buildRemoteStoreNodeAttributes(
-            segmentRepoName,
-            segmentRepoPath,
-            ReloadableFsRepository.TYPE,
-            translogRepoName,
-            translogRepoPath,
-            ReloadableFsRepository.TYPE,
-            withRateLimiterAttributes
-        );
-    }
-
-    public static Settings buildRemoteStoreNodeAttributes(
-        String segmentRepoName,
-        Path segmentRepoPath,
-        String segmentRepoType,
-        String translogRepoName,
-        Path translogRepoPath,
-        String translogRepoType,
-        boolean withRateLimiterAttributes
-    ) {
-        String segmentRepoTypeAttributeKey = String.format(
-            Locale.getDefault(),
-            "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT,
-            segmentRepoName
-        );
-        String segmentRepoSettingsAttributeKeyPrefix = String.format(
-            Locale.getDefault(),
-            "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX,
-            segmentRepoName
-        );
-        String translogRepoTypeAttributeKey = String.format(
-            Locale.getDefault(),
-            "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT,
-            translogRepoName
-        );
-        String translogRepoSettingsAttributeKeyPrefix = String.format(
-            Locale.getDefault(),
-            "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX,
-            translogRepoName
-        );
-        String stateRepoTypeAttributeKey = String.format(
-            Locale.getDefault(),
-            "node.attr." + REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT,
-            segmentRepoName
-        );
-        String stateRepoSettingsAttributeKeyPrefix = String.format(
-            Locale.getDefault(),
-            "node.attr." + REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX,
-            segmentRepoName
-        );
-
-        Settings.Builder settings = Settings.builder()
-            .put("node.attr." + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, segmentRepoName)
-            .put(segmentRepoTypeAttributeKey, segmentRepoType)
-            .put(segmentRepoSettingsAttributeKeyPrefix + "location", segmentRepoPath)
-            .put("node.attr." + REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, translogRepoName)
-            .put(translogRepoTypeAttributeKey, translogRepoType)
-            .put(translogRepoSettingsAttributeKeyPrefix + "location", translogRepoPath)
-            .put("node.attr." + REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY, segmentRepoName)
-            .put(stateRepoTypeAttributeKey, segmentRepoType)
-            .put(stateRepoSettingsAttributeKeyPrefix + "location", segmentRepoPath);
-
-        if (withRateLimiterAttributes) {
-            settings.put(segmentRepoSettingsAttributeKeyPrefix + "compress", randomBoolean())
-                .put(segmentRepoSettingsAttributeKeyPrefix + "chunk_size", 200, ByteSizeUnit.BYTES);
-        }
-
-        return settings.build();
-    }
-
     Settings defaultIndexSettings() {
         return Settings.builder()
             .put(super.indexSettings())
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java
index e1997fea3433a..ca0ae3ca9a700 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreIT.java
@@ -23,18 +23,22 @@
 import org.opensearch.cluster.routing.RecoverySource;
 import org.opensearch.cluster.routing.allocation.command.MoveAllocationCommand;
 import org.opensearch.common.Priority;
+import org.opensearch.common.blobstore.BlobPath;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.util.concurrent.BufferedAsyncIOProcessor;
 import org.opensearch.index.IndexSettings;
 import org.opensearch.index.shard.IndexShard;
 import org.opensearch.index.shard.IndexShardClosedException;
+import org.opensearch.index.translog.Translog;
 import org.opensearch.index.translog.Translog.Durability;
 import org.opensearch.indices.IndicesService;
+import org.opensearch.indices.RemoteStoreSettings;
 import org.opensearch.indices.recovery.PeerRecoveryTargetService;
 import org.opensearch.indices.recovery.RecoverySettings;
 import org.opensearch.indices.recovery.RecoveryState;
 import org.opensearch.plugins.Plugin;
+import org.opensearch.remotestore.multipart.mocks.MockFsRepositoryPlugin;
 import org.opensearch.test.InternalTestCluster;
 import org.opensearch.test.OpenSearchIntegTestCase;
 import org.opensearch.test.transport.MockTransportService;
@@ -56,7 +60,13 @@
 
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS;
-import static org.opensearch.indices.IndicesService.CLUSTER_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING;
+import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.SEGMENTS;
+import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.TRANSLOG;
+import static org.opensearch.index.remote.RemoteStoreEnums.DataType.DATA;
+import static org.opensearch.index.remote.RemoteStoreEnums.DataType.METADATA;
+import static org.opensearch.index.shard.IndexShardTestCase.getTranslog;
+import static org.opensearch.indices.RemoteStoreSettings.CLUSTER_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING;
+import static org.opensearch.test.OpenSearchTestCase.getShardLevelBlobPath;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount;
 import static org.hamcrest.Matchers.comparesEqualTo;
@@ -71,7 +81,7 @@ public class RemoteStoreIT extends RemoteStoreBaseIntegTestCase {
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Arrays.asList(MockTransportService.TestPlugin.class);
+        return Arrays.asList(MockTransportService.TestPlugin.class, MockFsRepositoryPlugin.class);
     }
 
     @Override
@@ -181,15 +191,11 @@ public void testStaleCommitDeletionWithInvokeFlush() throws Exception {
         createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 10000l, -1));
         int numberOfIterations = randomIntBetween(5, 15);
         indexData(numberOfIterations, true, INDEX_NAME);
-        String indexUUID = client().admin()
-            .indices()
-            .prepareGetSettings(INDEX_NAME)
-            .get()
-            .getSetting(INDEX_NAME, IndexMetadata.SETTING_INDEX_UUID);
-        Path indexPath = Path.of(String.valueOf(segmentRepoPath), indexUUID, "/0/segments/metadata");
-
+        String shardPath = getShardLevelBlobPath(client(), INDEX_NAME, BlobPath.cleanPath(), "0", SEGMENTS, METADATA).buildAsString();
+        Path indexPath = Path.of(segmentRepoPath + "/" + shardPath);
+        ;
         IndexShard indexShard = getIndexShard(dataNode, INDEX_NAME);
-        int lastNMetadataFilesToKeep = indexShard.getRecoverySettings().getMinRemoteSegmentMetadataFiles();
+        int lastNMetadataFilesToKeep = indexShard.getRemoteStoreSettings().getMinRemoteSegmentMetadataFiles();
         // Delete is async.
         assertBusy(() -> {
             int actualFileCount = getFileCount(indexPath);
@@ -211,12 +217,8 @@ public void testStaleCommitDeletionWithoutInvokeFlush() throws Exception {
         createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 10000l, -1));
         int numberOfIterations = randomIntBetween(5, 15);
         indexData(numberOfIterations, false, INDEX_NAME);
-        String indexUUID = client().admin()
-            .indices()
-            .prepareGetSettings(INDEX_NAME)
-            .get()
-            .getSetting(INDEX_NAME, IndexMetadata.SETTING_INDEX_UUID);
-        Path indexPath = Path.of(String.valueOf(segmentRepoPath), indexUUID, "/0/segments/metadata");
+        String shardPath = getShardLevelBlobPath(client(), INDEX_NAME, BlobPath.cleanPath(), "0", SEGMENTS, METADATA).buildAsString();
+        Path indexPath = Path.of(segmentRepoPath + "/" + shardPath);
         int actualFileCount = getFileCount(indexPath);
         // We also allow (numberOfIterations + 1) as index creation also triggers refresh.
         MatcherAssert.assertThat(actualFileCount, is(oneOf(numberOfIterations - 1, numberOfIterations, numberOfIterations + 1)));
@@ -224,18 +226,14 @@ public void testStaleCommitDeletionWithoutInvokeFlush() throws Exception {
 
     public void testStaleCommitDeletionWithMinSegmentFiles_3() throws Exception {
         Settings.Builder settings = Settings.builder()
-            .put(RecoverySettings.CLUSTER_REMOTE_INDEX_SEGMENT_METADATA_RETENTION_MAX_COUNT_SETTING.getKey(), "3");
+            .put(RemoteStoreSettings.CLUSTER_REMOTE_INDEX_SEGMENT_METADATA_RETENTION_MAX_COUNT_SETTING.getKey(), "3");
         internalCluster().startNode(settings);
 
         createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 10000l, -1));
         int numberOfIterations = randomIntBetween(5, 15);
         indexData(numberOfIterations, true, INDEX_NAME);
-        String indexUUID = client().admin()
-            .indices()
-            .prepareGetSettings(INDEX_NAME)
-            .get()
-            .getSetting(INDEX_NAME, IndexMetadata.SETTING_INDEX_UUID);
-        Path indexPath = Path.of(String.valueOf(segmentRepoPath), indexUUID, "/0/segments/metadata");
+        String shardPath = getShardLevelBlobPath(client(), INDEX_NAME, BlobPath.cleanPath(), "0", SEGMENTS, METADATA).buildAsString();
+        Path indexPath = Path.of(segmentRepoPath + "/" + shardPath);
         int actualFileCount = getFileCount(indexPath);
         // We also allow (numberOfIterations + 1) as index creation also triggers refresh.
         MatcherAssert.assertThat(actualFileCount, is(oneOf(4)));
@@ -243,18 +241,15 @@ public void testStaleCommitDeletionWithMinSegmentFiles_3() throws Exception {
 
     public void testStaleCommitDeletionWithMinSegmentFiles_Disabled() throws Exception {
         Settings.Builder settings = Settings.builder()
-            .put(RecoverySettings.CLUSTER_REMOTE_INDEX_SEGMENT_METADATA_RETENTION_MAX_COUNT_SETTING.getKey(), "-1");
+            .put(RemoteStoreSettings.CLUSTER_REMOTE_INDEX_SEGMENT_METADATA_RETENTION_MAX_COUNT_SETTING.getKey(), "-1");
         internalCluster().startNode(settings);
 
         createIndex(INDEX_NAME, remoteStoreIndexSettings(1, 10000l, -1));
         int numberOfIterations = randomIntBetween(12, 18);
         indexData(numberOfIterations, true, INDEX_NAME);
-        String indexUUID = client().admin()
-            .indices()
-            .prepareGetSettings(INDEX_NAME)
-            .get()
-            .getSetting(INDEX_NAME, IndexMetadata.SETTING_INDEX_UUID);
-        Path indexPath = Path.of(String.valueOf(segmentRepoPath), indexUUID, "/0/segments/metadata");
+        String shardPath = getShardLevelBlobPath(client(), INDEX_NAME, BlobPath.cleanPath(), "0", SEGMENTS, METADATA).buildAsString();
+        Path indexPath = Path.of(segmentRepoPath + "/" + shardPath);
+        ;
         int actualFileCount = getFileCount(indexPath);
         // We also allow (numberOfIterations + 1) as index creation also triggers refresh.
         MatcherAssert.assertThat(actualFileCount, is(oneOf(numberOfIterations + 1)));
@@ -469,7 +464,7 @@ public void testAsyncDurabilityThrowsExceptionWhenRestrictSettingTrue() throws E
 
     private void assertClusterRemoteBufferInterval(TimeValue expectedBufferInterval, String dataNode) {
         IndicesService indicesService = internalCluster().getInstance(IndicesService.class, dataNode);
-        assertEquals(expectedBufferInterval, indicesService.getClusterRemoteTranslogBufferInterval());
+        assertEquals(expectedBufferInterval, indicesService.getRemoteStoreSettings().getClusterRemoteTranslogBufferInterval());
     }
 
     private void assertBufferInterval(TimeValue expectedBufferInterval, IndexShard indexShard) {
@@ -588,12 +583,8 @@ public void testFallbackToNodeToNodeSegmentCopy() throws Exception {
         flushAndRefresh(INDEX_NAME);
 
         // 3. Delete data from remote segment store
-        String indexUUID = client().admin()
-            .indices()
-            .prepareGetSettings(INDEX_NAME)
-            .get()
-            .getSetting(INDEX_NAME, IndexMetadata.SETTING_INDEX_UUID);
-        Path segmentDataPath = Path.of(String.valueOf(segmentRepoPath), indexUUID, "/0/segments/data");
+        String shardPath = getShardLevelBlobPath(client(), INDEX_NAME, BlobPath.cleanPath(), "0", SEGMENTS, DATA).buildAsString();
+        Path segmentDataPath = Path.of(segmentRepoPath + "/" + shardPath);
 
         try (Stream<Path> files = Files.list(segmentDataPath)) {
             files.forEach(p -> {
@@ -802,4 +793,113 @@ public void testResumeUploadAfterFailedPrimaryRelocation() throws ExecutionExcep
             docs + moreDocs + uncommittedOps
         );
     }
+
+    // Test local only translog files which are not uploaded to remote store (no metadata present in remote)
+    // Without the cleanup change in RemoteFsTranslog.createEmptyTranslog, this test fails with NPE.
+    public void testLocalOnlyTranslogCleanupOnNodeRestart() throws Exception {
+        clusterSettingsSuppliedByTest = true;
+
+        // Overriding settings to use AsyncMultiStreamBlobContainer
+        Settings settings = Settings.builder()
+            .put(super.nodeSettings(1))
+            .put(
+                remoteStoreClusterSettings(
+                    REPOSITORY_NAME,
+                    segmentRepoPath,
+                    MockFsRepositoryPlugin.TYPE,
+                    REPOSITORY_2_NAME,
+                    translogRepoPath,
+                    MockFsRepositoryPlugin.TYPE
+                )
+            )
+            .build();
+
+        internalCluster().startClusterManagerOnlyNode(settings);
+        String dataNode = internalCluster().startDataOnlyNode(settings);
+
+        // 1. Create index with 0 replica
+        createIndex(INDEX_NAME, remoteStoreIndexSettings(0, 10000L, -1));
+        ensureGreen(INDEX_NAME);
+
+        // 2. Index docs
+        int searchableDocs = 0;
+        for (int i = 0; i < randomIntBetween(1, 5); i++) {
+            indexBulk(INDEX_NAME, 15);
+            refresh(INDEX_NAME);
+            searchableDocs += 15;
+        }
+        indexBulk(INDEX_NAME, 15);
+
+        assertHitCount(client(dataNode).prepareSearch(INDEX_NAME).setSize(0).get(), searchableDocs);
+
+        // 3. Delete metadata from remote translog
+        String indexUUID = client().admin()
+            .indices()
+            .prepareGetSettings(INDEX_NAME)
+            .get()
+            .getSetting(INDEX_NAME, IndexMetadata.SETTING_INDEX_UUID);
+
+        String shardPath = getShardLevelBlobPath(client(), INDEX_NAME, BlobPath.cleanPath(), "0", TRANSLOG, METADATA).buildAsString();
+        Path translogMetaDataPath = Path.of(translogRepoPath + "/" + shardPath);
+
+        try (Stream<Path> files = Files.list(translogMetaDataPath)) {
+            files.forEach(p -> {
+                try {
+                    Files.delete(p);
+                } catch (IOException e) {
+                    // Ignore
+                }
+            });
+        }
+
+        internalCluster().restartNode(dataNode);
+
+        ensureGreen(INDEX_NAME);
+
+        assertHitCount(client(dataNode).prepareSearch(INDEX_NAME).setSize(0).get(), searchableDocs);
+        indexBulk(INDEX_NAME, 15);
+        refresh(INDEX_NAME);
+        assertHitCount(client(dataNode).prepareSearch(INDEX_NAME).setSize(0).get(), searchableDocs + 15);
+    }
+
+    public void testFlushOnTooManyRemoteTranslogFiles() throws Exception {
+        internalCluster().startClusterManagerOnlyNode();
+        String datanode = internalCluster().startDataOnlyNodes(1).get(0);
+        createIndex(INDEX_NAME, remoteStoreIndexSettings(0, 10000L, -1));
+        ensureGreen(INDEX_NAME);
+
+        ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest();
+        updateSettingsRequest.persistentSettings(
+            Settings.builder().put(RemoteStoreSettings.CLUSTER_REMOTE_MAX_TRANSLOG_READERS.getKey(), "100")
+        );
+        assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet());
+
+        IndexShard indexShard = getIndexShard(datanode, INDEX_NAME);
+        Path translogLocation = getTranslog(indexShard).location();
+        assertFalse(indexShard.shouldPeriodicallyFlush());
+
+        try (Stream<Path> files = Files.list(translogLocation)) {
+            long totalFiles = files.filter(f -> f.getFileName().toString().endsWith(Translog.TRANSLOG_FILE_SUFFIX)).count();
+            assertEquals(totalFiles, 1L);
+        }
+
+        // indexing 100 documents (100 bulk requests), no flush will be triggered yet
+        for (int i = 0; i < 100; i++) {
+            indexBulk(INDEX_NAME, 1);
+        }
+
+        try (Stream<Path> files = Files.list(translogLocation)) {
+            long totalFiles = files.filter(f -> f.getFileName().toString().endsWith(Translog.TRANSLOG_FILE_SUFFIX)).count();
+            assertEquals(totalFiles, 101L);
+        }
+        // Will flush and trim the translog readers
+        indexBulk(INDEX_NAME, 1);
+
+        assertBusy(() -> {
+            try (Stream<Path> files = Files.list(translogLocation)) {
+                long totalFiles = files.filter(f -> f.getFileName().toString().endsWith(Translog.TRANSLOG_FILE_SUFFIX)).count();
+                assertEquals(totalFiles, 1L);
+            }
+        }, 30, TimeUnit.SECONDS);
+    }
 }
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRefreshListenerIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRefreshListenerIT.java
index acdb21d072320..65016c4976157 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRefreshListenerIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRefreshListenerIT.java
@@ -11,6 +11,7 @@
 import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse;
 import org.opensearch.action.admin.indices.stats.IndicesStatsRequest;
 import org.opensearch.action.admin.indices.stats.IndicesStatsResponse;
+import org.opensearch.common.blobstore.BlobPath;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.test.OpenSearchIntegTestCase;
 
@@ -22,7 +23,10 @@
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
+import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.SEGMENTS;
+import static org.opensearch.index.remote.RemoteStoreEnums.DataType.DATA;
 import static org.opensearch.index.remote.RemoteStorePressureSettings.REMOTE_REFRESH_SEGMENT_PRESSURE_ENABLED;
+import static org.opensearch.test.OpenSearchTestCase.getShardLevelBlobPath;
 
 @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
 public class RemoteStoreRefreshListenerIT extends AbstractRemoteStoreMockRepositoryIntegTestCase {
@@ -45,8 +49,10 @@ public void testRemoteRefreshRetryOnFailure() throws Exception {
         IndicesStatsResponse response = client().admin().indices().stats(new IndicesStatsRequest()).get();
         assertEquals(1, response.getShards().length);
 
+        String indexName = response.getShards()[0].getShardRouting().index().getName();
         String indexUuid = response.getShards()[0].getShardRouting().index().getUUID();
-        Path segmentDataRepoPath = location.resolve(String.format(Locale.ROOT, "%s/0/segments/data", indexUuid));
+        String shardPath = getShardLevelBlobPath(client(), indexName, new BlobPath(), "0", SEGMENTS, DATA).buildAsString();
+        Path segmentDataRepoPath = location.resolve(shardPath);
         String segmentDataLocalPath = String.format(Locale.ROOT, "%s/indices/%s/0/index", response.getShards()[0].getDataPath(), indexUuid);
 
         logger.info("--> Verify that the segment files are same on local and repository eventually");
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java
index 94acf2b1dbb27..bd84ab026dcea 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreRestoreIT.java
@@ -27,6 +27,7 @@
 import org.opensearch.repositories.fs.ReloadableFsRepository;
 import org.opensearch.test.InternalTestCluster;
 import org.opensearch.test.OpenSearchIntegTestCase;
+import org.opensearch.test.junit.annotations.TestIssueLogging;
 
 import java.io.IOException;
 import java.nio.file.Path;
@@ -42,7 +43,7 @@
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount;
 import static org.hamcrest.Matchers.greaterThan;
 
-@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.SUITE, numDataNodes = 0)
+@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
 public class RemoteStoreRestoreIT extends BaseRemoteStoreRestoreIT {
 
     /**
@@ -91,6 +92,7 @@ public void testRTSRestoreWithNoDataPostCommitPrimaryReplicaDown() throws Except
      * Simulates all data restored using Remote Translog Store.
      * @throws IOException IO Exception.
      */
+    @TestIssueLogging(value = "_root:TRACE", issueUrl = "https://github.com/opensearch-project/OpenSearch/issues/11085")
     public void testRTSRestoreWithNoDataPostRefreshPrimaryReplicaDown() throws Exception {
         testRestoreFlowBothPrimaryReplicasDown(1, false, true, randomIntBetween(1, 5));
     }
@@ -295,7 +297,6 @@ public void testRestoreFlowNoRedIndex() throws Exception {
      * for multiple indices matching a wildcard name pattern.
      * @throws IOException IO Exception.
      */
-    @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8480")
     public void testRTSRestoreWithCommittedDataMultipleIndicesPatterns() throws Exception {
         testRestoreFlowMultipleIndices(2, true, randomIntBetween(1, 5));
     }
@@ -306,16 +307,16 @@ public void testRTSRestoreWithCommittedDataMultipleIndicesPatterns() throws Exce
      * with all remote-enabled red indices considered for the restore by default.
      * @throws IOException IO Exception.
      */
-    @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8480")
     public void testRTSRestoreWithCommittedDataDefaultAllIndices() throws Exception {
         int shardCount = randomIntBetween(1, 5);
-        prepareCluster(1, 3, INDEX_NAMES, 1, shardCount);
+        int replicaCount = 1;
+        prepareCluster(1, 3, INDEX_NAMES, replicaCount, shardCount);
         String[] indices = INDEX_NAMES.split(",");
         Map<String, Map<String, Long>> indicesStats = new HashMap<>();
         for (String index : indices) {
             Map<String, Long> indexStats = indexData(2, true, index);
             indicesStats.put(index, indexStats);
-            assertEquals(shardCount, getNumShards(index).totalNumShards);
+            assertEquals(shardCount * (replicaCount + 1), getNumShards(index).totalNumShards);
         }
 
         for (String index : indices) {
@@ -337,7 +338,7 @@ public void testRTSRestoreWithCommittedDataDefaultAllIndices() throws Exception
         ensureGreen(indices);
 
         for (String index : indices) {
-            assertEquals(shardCount, getNumShards(index).totalNumShards);
+            assertEquals(shardCount * (replicaCount + 1), getNumShards(index).totalNumShards);
             verifyRestoredData(indicesStats.get(index), index);
         }
     }
@@ -395,16 +396,16 @@ public void testRTSRestoreWithCommittedDataNotAllRedRemoteIndices() throws Excep
      * except those matching the specified exclusion pattern.
      * @throws IOException IO Exception.
      */
-    @AwaitsFix(bugUrl = "https://github.com/opensearch-project/OpenSearch/issues/8480")
     public void testRTSRestoreWithCommittedDataExcludeIndicesPatterns() throws Exception {
         int shardCount = randomIntBetween(1, 5);
-        prepareCluster(1, 3, INDEX_NAMES, 1, shardCount);
+        int replicaCount = 1;
+        prepareCluster(1, 3, INDEX_NAMES, replicaCount, shardCount);
         String[] indices = INDEX_NAMES.split(",");
         Map<String, Map<String, Long>> indicesStats = new HashMap<>();
         for (String index : indices) {
             Map<String, Long> indexStats = indexData(2, true, index);
             indicesStats.put(index, indexStats);
-            assertEquals(shardCount, getNumShards(index).totalNumShards);
+            assertEquals(shardCount * (replicaCount + 1), getNumShards(index).totalNumShards);
         }
 
         for (String index : indices) {
@@ -433,9 +434,9 @@ public void testRTSRestoreWithCommittedDataExcludeIndicesPatterns() throws Excep
                 PlainActionFuture.newFuture()
             );
         ensureGreen(indices[0], indices[1]);
-        assertEquals(shardCount, getNumShards(indices[0]).totalNumShards);
+        assertEquals(shardCount * (replicaCount + 1), getNumShards(indices[0]).totalNumShards);
         verifyRestoredData(indicesStats.get(indices[0]), indices[0]);
-        assertEquals(shardCount, getNumShards(indices[1]).totalNumShards);
+        assertEquals(shardCount * (replicaCount + 1), getNumShards(indices[1]).totalNumShards);
         verifyRestoredData(indicesStats.get(indices[1]), indices[1]);
         ensureRed(indices[2], indices[3]);
     }
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreUploadIndexPathIT.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreUploadIndexPathIT.java
new file mode 100644
index 0000000000000..9b30dacfced13
--- /dev/null
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/RemoteStoreUploadIndexPathIT.java
@@ -0,0 +1,111 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.remotestore;
+
+import org.opensearch.action.admin.indices.delete.DeleteIndexRequest;
+import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.core.util.FileSystemUtils;
+import org.opensearch.index.remote.RemoteIndexPath;
+import org.opensearch.index.remote.RemoteIndexPathUploader;
+import org.opensearch.index.remote.RemoteStoreEnums;
+import org.opensearch.test.OpenSearchIntegTestCase;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.concurrent.ExecutionException;
+
+import static org.opensearch.gateway.remote.RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING;
+import static org.opensearch.indices.RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING;
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
+
+@OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0)
+public class RemoteStoreUploadIndexPathIT extends RemoteStoreBaseIntegTestCase {
+
+    private final String INDEX_NAME = "remote-store-test-idx-1";
+
+    @Override
+    protected Settings nodeSettings(int nodeOrdinal) {
+        return Settings.builder().put(super.nodeSettings(nodeOrdinal)).put(REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true).build();
+    }
+
+    /**
+     * Checks that the remote index path file gets created for the intended remote store path type and does not get created
+     * wherever not required.
+     */
+    public void testRemoteIndexPathFileCreation() throws ExecutionException, InterruptedException, IOException {
+        String clusterManagerNode = internalCluster().startClusterManagerOnlyNode();
+        internalCluster().startDataOnlyNodes(2);
+
+        // Case 1 - Hashed_prefix, we would need the remote index path file to be created.
+        client(clusterManagerNode).admin()
+            .cluster()
+            .prepareUpdateSettings()
+            .setTransientSettings(
+                Settings.builder().put(CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), RemoteStoreEnums.PathType.HASHED_PREFIX)
+            )
+            .get();
+
+        createIndex(INDEX_NAME, remoteStoreIndexSettings(0, 1));
+        validateRemoteIndexPathFile(true);
+        assertAcked(client().admin().indices().delete(new DeleteIndexRequest(INDEX_NAME)).get());
+        FileSystemUtils.deleteSubDirectories(translogRepoPath);
+        FileSystemUtils.deleteSubDirectories(segmentRepoPath);
+
+        // Case 2 - Hashed_infix, we would not have the remote index path file created here.
+        client(clusterManagerNode).admin()
+            .cluster()
+            .prepareUpdateSettings()
+            .setTransientSettings(
+                Settings.builder().put(CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), RemoteStoreEnums.PathType.HASHED_INFIX)
+            )
+            .get();
+        createIndex(INDEX_NAME, remoteStoreIndexSettings(0, 1));
+        validateRemoteIndexPathFile(false);
+        assertAcked(client().admin().indices().delete(new DeleteIndexRequest(INDEX_NAME)).get());
+
+        // Case 3 - fixed, we would not have the remote index path file created here either.
+        client(clusterManagerNode).admin()
+            .cluster()
+            .prepareUpdateSettings()
+            .setTransientSettings(Settings.builder().put(CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), RemoteStoreEnums.PathType.FIXED))
+            .get();
+        createIndex(INDEX_NAME, remoteStoreIndexSettings(0, 1));
+        validateRemoteIndexPathFile(false);
+        assertAcked(client().admin().indices().delete(new DeleteIndexRequest(INDEX_NAME)).get());
+
+    }
+
+    private void validateRemoteIndexPathFile(boolean exists) throws IOException {
+        String indexUUID = client().admin()
+            .indices()
+            .prepareGetSettings(INDEX_NAME)
+            .get()
+            .getSetting(INDEX_NAME, IndexMetadata.SETTING_INDEX_UUID);
+        String fileName = generatePartFileName(indexUUID);
+        assertEquals(exists, FileSystemUtils.exists(translogRepoPath.resolve(RemoteIndexPath.DIR)));
+        if (exists) {
+            Path[] files = FileSystemUtils.files(translogRepoPath.resolve(RemoteIndexPath.DIR));
+            assertEquals(1, files.length);
+            assertTrue(Arrays.stream(files).anyMatch(file -> file.toString().contains(fileName)));
+            String translogPathFile = files[0].toString();
+            assertTrue(FileSystemUtils.exists(segmentRepoPath.resolve(RemoteIndexPath.DIR)));
+            files = FileSystemUtils.files(segmentRepoPath.resolve(RemoteIndexPath.DIR));
+            assertEquals(1, files.length);
+            assertTrue(Arrays.stream(files).anyMatch(file -> file.toString().contains(fileName)));
+            String segmentPathFile = files[0].toString();
+            assertNotEquals(translogPathFile, segmentPathFile);
+        }
+    }
+
+    private String generatePartFileName(String indexUUID) {
+        return String.join(RemoteIndexPathUploader.DELIMITER, indexUUID, "2", RemoteIndexPath.DEFAULT_VERSION);
+    }
+}
diff --git a/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsAsyncBlobContainer.java b/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsAsyncBlobContainer.java
index 36987ac2d4991..d45b4e3deb798 100644
--- a/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsAsyncBlobContainer.java
+++ b/server/src/internalClusterTest/java/org/opensearch/remotestore/multipart/mocks/MockFsAsyncBlobContainer.java
@@ -131,7 +131,7 @@ public void readBlobAsync(String blobName, ActionListener<ReadContext> listener)
                     InputStreamContainer blobPartStream = new InputStreamContainer(readBlob(blobName, offset, partSize), partSize, offset);
                     blobPartStreams.add(() -> CompletableFuture.completedFuture(blobPartStream));
                 }
-                ReadContext blobReadContext = new ReadContext(contentLength, blobPartStreams, null);
+                ReadContext blobReadContext = new ReadContext.Builder(contentLength, blobPartStreams).build();
                 listener.onResponse(blobReadContext);
             } catch (Exception e) {
                 listener.onFailure(e);
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/AggregationsIntegrationIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/AggregationsIntegrationIT.java
index 6059abce53c8b..4a8b00ea45738 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/AggregationsIntegrationIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/AggregationsIntegrationIT.java
@@ -38,6 +38,8 @@
 import org.opensearch.action.index.IndexRequestBuilder;
 import org.opensearch.action.search.SearchPhaseExecutionException;
 import org.opensearch.action.search.SearchResponse;
+import org.opensearch.action.support.WriteRequest;
+import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.search.aggregations.bucket.terms.IncludeExclude;
@@ -56,6 +58,8 @@
 import java.util.List;
 
 import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING;
+import static org.opensearch.search.aggregations.AggregationBuilders.global;
+import static org.opensearch.search.aggregations.AggregationBuilders.stats;
 import static org.opensearch.search.aggregations.AggregationBuilders.terms;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse;
@@ -164,4 +168,23 @@ private void runLargeStringAggregationTest(AggregationBuilder aggregation) {
         }
         assertTrue("Exception should have been thrown", exceptionThrown);
     }
+
+    public void testAggsOnEmptyShards() {
+        // Create index with 5 shards but only 1 doc
+        assertAcked(
+            prepareCreate(
+                "idx",
+                Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 5).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+            ).setMapping("score", "type=integer")
+        );
+        client().prepareIndex("idx").setId("1").setSource("score", "5").setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE).get();
+
+        // Validate global agg does not throw an exception
+        assertSearchResponse(
+            client().prepareSearch("idx").addAggregation(global("global").subAggregation(stats("value_stats").field("score"))).get()
+        );
+
+        // Validate non-global agg does not throw an exception
+        assertSearchResponse(client().prepareSearch("idx").addAggregation(stats("value_stats").field("score")).get());
+    }
 }
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/CompositeAggIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/CompositeAggIT.java
index 5a38ba670f1dc..a743f22a2ff77 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/CompositeAggIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/CompositeAggIT.java
@@ -26,6 +26,7 @@
 import java.util.Collection;
 import java.util.List;
 
+import static org.opensearch.indices.IndicesRequestCache.INDEX_CACHE_REQUEST_ENABLED_SETTING;
 import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertSearchResponse;
@@ -50,23 +51,25 @@ public void setupSuiteScopeCluster() throws Exception {
         assertAcked(
             prepareCreate(
                 "idx",
-                Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                Settings.builder()
+                    .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                    .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+                    .put(INDEX_CACHE_REQUEST_ENABLED_SETTING.getKey(), false)
             ).setMapping("type", "type=keyword", "num", "type=integer", "score", "type=integer")
         );
         waitForRelocation(ClusterHealthStatus.GREEN);
 
-        client().prepareIndex("idx").setId("1").setSource("type", "type1", "num", "1", "score", "5").get();
-        client().prepareIndex("idx").setId("1").setSource("type", "type2", "num", "11", "score", "50").get();
-        refresh("idx");
-        client().prepareIndex("idx").setId("1").setSource("type", "type1", "num", "1", "score", "2").get();
-        client().prepareIndex("idx").setId("1").setSource("type", "type2", "num", "12", "score", "20").get();
-        refresh("idx");
-        client().prepareIndex("idx").setId("1").setSource("type", "type1", "num", "3", "score", "10").get();
-        client().prepareIndex("idx").setId("1").setSource("type", "type2", "num", "13", "score", "15").get();
-        refresh("idx");
-        client().prepareIndex("idx").setId("1").setSource("type", "type1", "num", "3", "score", "1").get();
-        client().prepareIndex("idx").setId("1").setSource("type", "type2", "num", "13", "score", "100").get();
-        refresh("idx");
+        indexRandom(
+            true,
+            client().prepareIndex("idx").setId("1").setSource("type", "type1", "num", "1", "score", "5"),
+            client().prepareIndex("idx").setId("1").setSource("type", "type2", "num", "11", "score", "50"),
+            client().prepareIndex("idx").setId("1").setSource("type", "type1", "num", "1", "score", "2"),
+            client().prepareIndex("idx").setId("1").setSource("type", "type2", "num", "12", "score", "20"),
+            client().prepareIndex("idx").setId("1").setSource("type", "type1", "num", "3", "score", "10"),
+            client().prepareIndex("idx").setId("1").setSource("type", "type2", "num", "13", "score", "15"),
+            client().prepareIndex("idx").setId("1").setSource("type", "type1", "num", "3", "score", "1"),
+            client().prepareIndex("idx").setId("1").setSource("type", "type2", "num", "13", "score", "100")
+        );
 
         waitForRelocation(ClusterHealthStatus.GREEN);
         refresh();
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/FilterRewriteIT.java b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/FilterRewriteIT.java
index e051265d4b3bc..3f32cf3bfbbe7 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/FilterRewriteIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/aggregations/bucket/FilterRewriteIT.java
@@ -10,6 +10,7 @@
 
 import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
 
+import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse;
 import org.opensearch.action.index.IndexRequestBuilder;
 import org.opensearch.action.search.SearchResponse;
 import org.opensearch.common.settings.Settings;
@@ -34,6 +35,7 @@
 
 import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder;
 import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING;
+import static org.opensearch.search.SearchService.MAX_AGGREGATION_REWRITE_FILTERS;
 import static org.opensearch.search.aggregations.AggregationBuilders.dateHistogram;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
 
@@ -93,7 +95,7 @@ public void testMinDocCountOnDateHistogram() throws Exception {
         final SearchResponse allResponse = client().prepareSearch("idx")
             .setSize(0)
             .setQuery(QUERY)
-            .addAggregation(dateHistogram("histo").field("date").dateHistogramInterval(DateHistogramInterval.DAY).minDocCount(0))
+            .addAggregation(dateHistogram("histo").field("date").calendarInterval(DateHistogramInterval.DAY).minDocCount(0))
             .get();
 
         final Histogram allHisto = allResponse.getAggregations().get("histo");
@@ -104,4 +106,36 @@ public void testMinDocCountOnDateHistogram() throws Exception {
             assertEquals(entry.getValue(), results.get(entry.getKey()));
         }
     }
+
+    public void testDisableOptimizationGivesSameResults() throws Exception {
+        SearchResponse response = client().prepareSearch("idx")
+            .setSize(0)
+            .addAggregation(dateHistogram("histo").field("date").calendarInterval(DateHistogramInterval.DAY).minDocCount(0))
+            .get();
+
+        final Histogram allHisto1 = response.getAggregations().get("histo");
+
+        final ClusterUpdateSettingsResponse updateSettingResponse = client().admin()
+            .cluster()
+            .prepareUpdateSettings()
+            .setTransientSettings(Settings.builder().put(MAX_AGGREGATION_REWRITE_FILTERS.getKey(), 0))
+            .get();
+
+        assertEquals(updateSettingResponse.getTransientSettings().get(MAX_AGGREGATION_REWRITE_FILTERS.getKey()), "0");
+
+        response = client().prepareSearch("idx")
+            .setSize(0)
+            .addAggregation(dateHistogram("histo").field("date").calendarInterval(DateHistogramInterval.DAY).minDocCount(0))
+            .get();
+
+        final Histogram allHisto2 = response.getAggregations().get("histo");
+
+        assertEquals(allHisto1, allHisto2);
+
+        client().admin()
+            .cluster()
+            .prepareUpdateSettings()
+            .setTransientSettings(Settings.builder().putNull(MAX_AGGREGATION_REWRITE_FILTERS.getKey()))
+            .get();
+    }
 }
diff --git a/server/src/internalClusterTest/java/org/opensearch/search/fields/SearchFieldsIT.java b/server/src/internalClusterTest/java/org/opensearch/search/fields/SearchFieldsIT.java
index 906d45ef84b3f..2ce96092203e8 100644
--- a/server/src/internalClusterTest/java/org/opensearch/search/fields/SearchFieldsIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/search/fields/SearchFieldsIT.java
@@ -40,6 +40,7 @@
 import org.opensearch.common.Numbers;
 import org.opensearch.common.collect.MapBuilder;
 import org.opensearch.common.document.DocumentField;
+import org.opensearch.common.geo.GeoPoint;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.time.DateFormatter;
 import org.opensearch.common.time.DateUtils;
@@ -51,6 +52,7 @@
 import org.opensearch.core.xcontent.MediaTypeRegistry;
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.index.fielddata.ScriptDocValues;
+import org.opensearch.index.mapper.DateFieldMapper;
 import org.opensearch.index.mapper.MapperService;
 import org.opensearch.index.query.QueryBuilders;
 import org.opensearch.plugins.Plugin;
@@ -189,6 +191,20 @@ protected Map<String, Function<Map<String, Object>, Object>> pluginScripts() {
             scripts.put("doc['s']", vars -> docScript(vars, "s"));
             scripts.put("doc['ms']", vars -> docScript(vars, "ms"));
 
+            scripts.put("doc['keyword_field']", vars -> sourceScript(vars, "keyword_field"));
+            scripts.put("doc['multi_keyword_field']", vars -> sourceScript(vars, "multi_keyword_field"));
+            scripts.put("doc['long_field']", vars -> sourceScript(vars, "long_field"));
+            scripts.put("doc['multi_long_field']", vars -> sourceScript(vars, "multi_long_field"));
+            scripts.put("doc['double_field']", vars -> sourceScript(vars, "double_field"));
+            scripts.put("doc['multi_double_field']", vars -> sourceScript(vars, "multi_double_field"));
+            scripts.put("doc['date_field']", vars -> sourceScript(vars, "date_field"));
+            scripts.put("doc['multi_date_field']", vars -> sourceScript(vars, "multi_date_field"));
+            scripts.put("doc['ip_field']", vars -> sourceScript(vars, "ip_field"));
+            scripts.put("doc['multi_ip_field']", vars -> sourceScript(vars, "multi_ip_field"));
+            scripts.put("doc['boolean_field']", vars -> sourceScript(vars, "boolean_field"));
+            scripts.put("doc['geo_field']", vars -> sourceScript(vars, "geo_field"));
+            scripts.put("doc['multi_geo_field']", vars -> sourceScript(vars, "multi_geo_field"));
+
             return scripts;
         }
 
@@ -1299,6 +1315,147 @@ public void testScriptFields() throws Exception {
         }
     }
 
+    public void testDerivedFields() throws Exception {
+        assertAcked(
+            prepareCreate("index").setMapping(
+                "keyword_field",
+                "type=keyword",
+                "multi_keyword_field",
+                "type=keyword",
+                "long_field",
+                "type=long",
+                "multi_long_field",
+                "type=long",
+                "double_field",
+                "type=double",
+                "multi_double_field",
+                "type=double",
+                "date_field",
+                "type=date",
+                "multi_date_field",
+                "type=date",
+                "ip_field",
+                "type=ip",
+                "multi_ip_field",
+                "type=ip",
+                "boolean_field",
+                "type=boolean",
+                "geo_field",
+                "type=geo_point",
+                "multi_geo_field",
+                "type=geo_point"
+            ).get()
+        );
+        final int numDocs = randomIntBetween(3, 8);
+        List<IndexRequestBuilder> reqs = new ArrayList<>();
+
+        DateTime date1 = new DateTime(1990, 12, 29, 0, 0, DateTimeZone.UTC);
+        DateTime date2 = new DateTime(1990, 12, 30, 0, 0, DateTimeZone.UTC);
+
+        for (int i = 0; i < numDocs; ++i) {
+            reqs.add(
+                client().prepareIndex("index")
+                    .setId(Integer.toString(i))
+                    .setSource(
+                        "keyword_field",
+                        Integer.toString(i),
+                        "multi_keyword_field",
+                        new String[] { Integer.toString(i), Integer.toString(i + 1) },
+                        "long_field",
+                        (long) i,
+                        "multi_long_field",
+                        new long[] { i, i + 1 },
+                        "double_field",
+                        (double) i,
+                        "multi_double_field",
+                        new double[] { i, i + 1 },
+                        "date_field",
+                        date1.getMillis(),
+                        "multi_date_field",
+                        new Long[] { date1.getMillis(), date2.getMillis() },
+                        "ip_field",
+                        "172.16.1.10",
+                        "multi_ip_field",
+                        new String[] { "172.16.1.10", "172.16.1.11" },
+                        "boolean_field",
+                        true,
+                        "geo_field",
+                        new GeoPoint(12.0, 10.0),
+                        "multi_geo_field",
+                        new GeoPoint[] { new GeoPoint(12.0, 10.0), new GeoPoint(13.0, 10.0) }
+                    )
+            );
+        }
+        indexRandom(true, reqs);
+        indexRandomForConcurrentSearch("index");
+        ensureSearchable();
+        SearchRequestBuilder req = client().prepareSearch("index");
+        String[][] fieldLookup = new String[][] {
+            { "keyword_field", "keyword" },
+            { "multi_keyword_field", "keyword" },
+            { "long_field", "long" },
+            { "multi_long_field", "long" },
+            { "double_field", "double" },
+            { "multi_double_field", "double" },
+            { "date_field", "date" },
+            { "multi_date_field", "date" },
+            { "ip_field", "ip" },
+            { "multi_ip_field", "ip" },
+            { "boolean_field", "boolean" },
+            { "geo_field", "geo_point" },
+            { "multi_geo_field", "geo_point" } };
+        for (String[] field : fieldLookup) {
+            req.addDerivedField(
+                "derived_" + field[0],
+                field[1],
+                new Script(ScriptType.INLINE, CustomScriptPlugin.NAME, "doc['" + field[0] + "']", Collections.emptyMap())
+            );
+        }
+        req.addFetchField("derived_*");
+        SearchResponse resp = req.get();
+        assertSearchResponse(resp);
+        for (SearchHit hit : resp.getHits().getHits()) {
+            final int id = Integer.parseInt(hit.getId());
+            Map<String, DocumentField> fields = hit.getFields();
+
+            assertEquals(fields.get("derived_keyword_field").getValues().get(0), Integer.toString(id));
+            assertEquals(fields.get("derived_multi_keyword_field").getValues().get(0), Integer.toString(id));
+            assertEquals(fields.get("derived_multi_keyword_field").getValues().get(1), Integer.toString(id + 1));
+
+            assertEquals(fields.get("derived_long_field").getValues().get(0), id);
+            assertEquals(fields.get("derived_multi_long_field").getValues().get(0), id);
+            assertEquals(fields.get("derived_multi_long_field").getValues().get(1), (id + 1));
+
+            assertEquals(fields.get("derived_double_field").getValues().get(0), (double) id);
+            assertEquals(fields.get("derived_multi_double_field").getValues().get(0), (double) id);
+            assertEquals(fields.get("derived_multi_double_field").getValues().get(1), (double) (id + 1));
+
+            assertEquals(
+                fields.get("derived_date_field").getValues().get(0),
+                DateFieldMapper.getDefaultDateTimeFormatter().formatJoda(date1)
+            );
+            assertEquals(
+                fields.get("derived_multi_date_field").getValues().get(0),
+                DateFieldMapper.getDefaultDateTimeFormatter().formatJoda(date1)
+            );
+            assertEquals(
+                fields.get("derived_multi_date_field").getValues().get(1),
+                DateFieldMapper.getDefaultDateTimeFormatter().formatJoda(date2)
+            );
+
+            assertEquals(fields.get("derived_ip_field").getValues().get(0), "172.16.1.10");
+            assertEquals(fields.get("derived_multi_ip_field").getValues().get(0), "172.16.1.10");
+            assertEquals(fields.get("derived_multi_ip_field").getValues().get(1), "172.16.1.11");
+
+            assertEquals(fields.get("derived_boolean_field").getValues().get(0), true);
+
+            assertEquals(fields.get("derived_geo_field").getValues().get(0), new GeoPoint(12.0, 10.0));
+            assertEquals(fields.get("derived_multi_geo_field").getValues().get(0), new GeoPoint(12.0, 10.0));
+            assertEquals(fields.get("derived_multi_geo_field").getValues().get(1), new GeoPoint(13.0, 10.0));
+
+        }
+    }
+
     public void testDocValueFieldsWithFieldAlias() throws Exception {
         XContentBuilder mapping = XContentFactory.jsonBuilder()
             .startObject()
diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/DedicatedClusterSnapshotRestoreIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/DedicatedClusterSnapshotRestoreIT.java
index 7a52c8aa5018e..54db951eb41c2 100644
--- a/server/src/internalClusterTest/java/org/opensearch/snapshots/DedicatedClusterSnapshotRestoreIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/DedicatedClusterSnapshotRestoreIT.java
@@ -572,7 +572,7 @@ public void testRestoreIndexWithMissingShards() throws Exception {
                 List<SnapshotStatus> snapshotStatuses = snapshotsStatusResponse.getSnapshots();
                 assertEquals(snapshotStatuses.size(), 1);
                 logger.trace("current snapshot status [{}]", snapshotStatuses.get(0));
-                assertTrue(snapshotStatuses.get(0).getState().completed());
+                assertThat(getSnapshot("test-repo", "test-snap-2").state(), equalTo(SnapshotState.PARTIAL));
             }, 1, TimeUnit.MINUTES);
             SnapshotsStatusResponse snapshotsStatusResponse = clusterAdmin().prepareSnapshotStatus("test-repo")
                 .setSnapshots("test-snap-2")
@@ -589,7 +589,6 @@ public void testRestoreIndexWithMissingShards() throws Exception {
             // After it was marked as completed in the cluster state - we need to check if it's completed on the file system as well
             assertBusy(() -> {
                 SnapshotInfo snapshotInfo = getSnapshot("test-repo", "test-snap-2");
-                assertTrue(snapshotInfo.state().completed());
                 assertEquals(SnapshotState.PARTIAL, snapshotInfo.state());
             }, 1, TimeUnit.MINUTES);
         } else {
diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java
index 78827849a8037..e688a4491b1a7 100644
--- a/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/DeleteSnapshotIT.java
@@ -14,9 +14,14 @@
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.common.UUIDs;
 import org.opensearch.common.action.ActionFuture;
+import org.opensearch.common.blobstore.BlobContainer;
+import org.opensearch.common.blobstore.BlobPath;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
+import org.opensearch.index.store.RemoteBufferedOutputDirectory;
 import org.opensearch.remotestore.RemoteStoreBaseIntegTestCase;
+import org.opensearch.repositories.RepositoriesService;
+import org.opensearch.repositories.blobstore.BlobStoreRepository;
 import org.opensearch.test.OpenSearchIntegTestCase;
 
 import java.nio.file.Path;
@@ -27,6 +32,8 @@
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Stream;
 
+import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.SEGMENTS;
+import static org.opensearch.index.remote.RemoteStoreEnums.DataType.LOCK_FILES;
 import static org.opensearch.remotestore.RemoteStoreBaseIntegTestCase.remoteStoreClusterSettings;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.comparesEqualTo;
@@ -307,7 +314,21 @@ public void testRemoteStoreCleanupForDeletedIndex() throws Exception {
         SnapshotInfo snapshotInfo1 = createFullSnapshot(snapshotRepoName, "snap1");
         SnapshotInfo snapshotInfo2 = createFullSnapshot(snapshotRepoName, "snap2");
 
-        String[] lockFiles = getLockFilesInRemoteStore(remoteStoreEnabledIndexName, REMOTE_REPO_NAME);
+        final RepositoriesService repositoriesService = internalCluster().getCurrentClusterManagerNodeInstance(RepositoriesService.class);
+        final BlobStoreRepository remoteStoreRepository = (BlobStoreRepository) repositoriesService.repository(REMOTE_REPO_NAME);
+        BlobPath shardLevelBlobPath = getShardLevelBlobPath(
+            client(),
+            remoteStoreEnabledIndexName,
+            remoteStoreRepository.basePath(),
+            "0",
+            SEGMENTS,
+            LOCK_FILES
+        );
+        BlobContainer blobContainer = remoteStoreRepository.blobStore().blobContainer(shardLevelBlobPath);
+        String[] lockFiles;
+        try (RemoteBufferedOutputDirectory lockDirectory = new RemoteBufferedOutputDirectory(blobContainer)) {
+            lockFiles = lockDirectory.listAll();
+        }
         assert (lockFiles.length == 2) : "lock files are " + Arrays.toString(lockFiles);
 
         // delete remote store index
@@ -320,7 +341,9 @@ public void testRemoteStoreCleanupForDeletedIndex() throws Exception {
             .get();
         assertAcked(deleteSnapshotResponse);
 
-        lockFiles = getLockFilesInRemoteStore(remoteStoreEnabledIndexName, REMOTE_REPO_NAME, indexUUID);
+        try (RemoteBufferedOutputDirectory lockDirectory = new RemoteBufferedOutputDirectory(blobContainer)) {
+            lockFiles = lockDirectory.listAll();
+        }
         assert (lockFiles.length == 1) : "lock files are " + Arrays.toString(lockFiles);
         assertTrue(lockFiles[0].contains(snapshotInfo2.snapshotId().getUUID()));
 
diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/SegmentReplicationSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/SegmentReplicationSnapshotIT.java
index b019bb57743c9..df1fc9b833171 100644
--- a/server/src/internalClusterTest/java/org/opensearch/snapshots/SegmentReplicationSnapshotIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/SegmentReplicationSnapshotIT.java
@@ -112,12 +112,16 @@ public void createSnapshot() {
     }
 
     public RestoreSnapshotResponse restoreSnapshotWithSettings(Settings indexSettings) {
+        return restoreSnapshotWithSettings(indexSettings, RESTORED_INDEX_NAME);
+    }
+
+    public RestoreSnapshotResponse restoreSnapshotWithSettings(Settings indexSettings, String restoredIndexName) {
         RestoreSnapshotRequestBuilder builder = client().admin()
             .cluster()
             .prepareRestoreSnapshot(REPOSITORY_NAME, SNAPSHOT_NAME)
             .setWaitForCompletion(false)
             .setRenamePattern(INDEX_NAME)
-            .setRenameReplacement(RESTORED_INDEX_NAME);
+            .setRenameReplacement(restoredIndexName);
         if (indexSettings != null) {
             builder.setIndexSettings(indexSettings);
         }
@@ -311,7 +315,8 @@ public void testSnapshotRestoreOnIndexWithSegRepClusterSetting() throws Exceptio
     * 2. Snapshot index
     * 3. Add new set of nodes with `cluster.indices.replication.strategy` set to SEGMENT and `cluster.index.restrict.replication.type`
     *    set to true.
-    * 4. Perform restore on new set of nodes to validate restored index has `DOCUMENT` replication.
+    * 4. Perform restore on new set of nodes to validate restored index has `SEGMENT` replication.
+    * 5. Validate that if replication type is passed as DOCUMENT as request parameter, restore operation fails
     */
     public void testSnapshotRestoreOnRestrictReplicationSetting() throws Exception {
         final int documentCount = scaledRandomIntBetween(1, 10);
@@ -337,9 +342,20 @@ public void testSnapshotRestoreOnRestrictReplicationSetting() throws Exception {
 
         createSnapshot();
 
-        // Delete index
+        RestoreSnapshotResponse restoreSnapshotResponse = restoreSnapshotWithSettings(restoreIndexSegRepSettings(), RESTORED_INDEX_NAME);
+        assertEquals(restoreSnapshotResponse.status(), RestStatus.ACCEPTED);
+        ensureGreen(RESTORED_INDEX_NAME);
+        GetSettingsResponse settingsResponse = client().admin()
+            .indices()
+            .getSettings(new GetSettingsRequest().indices(RESTORED_INDEX_NAME).includeDefaults(true))
+            .get();
+        assertEquals(settingsResponse.getSetting(RESTORED_INDEX_NAME, SETTING_REPLICATION_TYPE), ReplicationType.SEGMENT.toString());
+
+        // Delete indices
         assertAcked(client().admin().indices().delete(new DeleteIndexRequest(INDEX_NAME)).get());
         assertFalse("index [" + INDEX_NAME + "] should have been deleted", indexExists(INDEX_NAME));
+        assertAcked(client().admin().indices().delete(new DeleteIndexRequest(RESTORED_INDEX_NAME)).get());
+        assertFalse("index [" + RESTORED_INDEX_NAME + "] should have been deleted", indexExists(RESTORED_INDEX_NAME));
 
         // Start new set of nodes with cluster level replication type setting and restrict replication type setting.
         Settings settings = Settings.builder()
@@ -361,7 +377,25 @@ public void testSnapshotRestoreOnRestrictReplicationSetting() throws Exception {
         // Perform snapshot restore
         logger.info("--> Performing snapshot restore to target index");
 
-        IllegalArgumentException exception = expectThrows(IllegalArgumentException.class, () -> restoreSnapshotWithSettings(null));
+        restoreSnapshotResponse = restoreSnapshotWithSettings(null);
+
+        // Assertions
+        assertEquals(restoreSnapshotResponse.status(), RestStatus.ACCEPTED);
+        ensureGreen(RESTORED_INDEX_NAME);
+        settingsResponse = client().admin()
+            .indices()
+            .getSettings(new GetSettingsRequest().indices(RESTORED_INDEX_NAME).includeDefaults(true))
+            .get();
+        assertEquals(settingsResponse.getSetting(RESTORED_INDEX_NAME, SETTING_REPLICATION_TYPE), ReplicationType.SEGMENT.toString());
+
+        // restore index with cluster default setting
+        restoreSnapshotWithSettings(restoreIndexSegRepSettings(), RESTORED_INDEX_NAME + "1");
+
+        // Perform Snapshot Restore with different index name
+        IllegalArgumentException exception = expectThrows(
+            IllegalArgumentException.class,
+            () -> restoreSnapshotWithSettings(restoreIndexDocRepSettings(), RESTORED_INDEX_NAME + "2")
+        );
         assertEquals(REPLICATION_MISMATCH_VALIDATION_ERROR, exception.getMessage());
     }
 }
diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java
index c574233d25051..fb69209f7adda 100644
--- a/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java
+++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java
@@ -40,7 +40,6 @@
 import org.opensearch.action.admin.cluster.snapshots.status.SnapshotIndexShardStatus;
 import org.opensearch.action.admin.cluster.snapshots.status.SnapshotStats;
 import org.opensearch.action.admin.cluster.snapshots.status.SnapshotStatus;
-import org.opensearch.action.admin.cluster.snapshots.status.SnapshotsStatusRequest;
 import org.opensearch.action.admin.cluster.snapshots.status.SnapshotsStatusResponse;
 import org.opensearch.client.Client;
 import org.opensearch.cluster.SnapshotsInProgress;
@@ -101,13 +100,9 @@ public void testStatusApiConsistency() {
         assertThat(snapshotInfo.state(), equalTo(SnapshotState.SUCCESS));
         assertThat(snapshotInfo.version(), equalTo(Version.CURRENT));
 
-        final List<SnapshotStatus> snapshotStatus = clusterAdmin().snapshotsStatus(
-            new SnapshotsStatusRequest("test-repo", new String[] { "test-snap" })
-        ).actionGet().getSnapshots();
-        assertThat(snapshotStatus.size(), equalTo(1));
-        final SnapshotStatus snStatus = snapshotStatus.get(0);
-        assertEquals(snStatus.getStats().getStartTime(), snapshotInfo.startTime());
-        assertEquals(snStatus.getStats().getTime(), snapshotInfo.endTime() - snapshotInfo.startTime());
+        final SnapshotStatus snapshotStatus = getSnapshotStatus("test-repo", "test-snap");
+        assertEquals(snapshotStatus.getStats().getStartTime(), snapshotInfo.startTime());
+        assertEquals(snapshotStatus.getStats().getTime(), snapshotInfo.endTime() - snapshotInfo.startTime());
     }
 
     public void testStatusAPICallForShallowCopySnapshot() {
@@ -357,6 +352,22 @@ public void testSnapshotStatusOnFailedSnapshot() throws Exception {
         assertEquals(SnapshotsInProgress.State.FAILED, snapshotsStatusResponse.getSnapshots().get(0).getState());
     }
 
+    public void testSnapshotStatusOnPartialSnapshot() throws Exception {
+        final String dataNode = internalCluster().startDataOnlyNode();
+        final String repoName = "test-repo";
+        final String snapshotName = "test-snap";
+        final String indexName = "test-idx";
+        createRepository(repoName, "fs");
+        // create an index with a single shard on the data node, that will be stopped
+        createIndex(indexName, singleShardOneNode(dataNode));
+        index(indexName, "_doc", "some_doc_id", "foo", "bar");
+        logger.info("--> stopping data node before creating snapshot");
+        stopNode(dataNode);
+        startFullSnapshot(repoName, snapshotName, true).get();
+        final SnapshotStatus snapshotStatus = getSnapshotStatus(repoName, snapshotName);
+        assertEquals(SnapshotsInProgress.State.PARTIAL, snapshotStatus.getState());
+    }
+
     public void testStatusAPICallInProgressShallowSnapshot() throws Exception {
         internalCluster().startClusterManagerOnlyNode();
         internalCluster().startDataOnlyNode();
diff --git a/server/src/main/java/org/opensearch/action/ActionModule.java b/server/src/main/java/org/opensearch/action/ActionModule.java
index f827b7f3f0097..5e2b62614fc47 100644
--- a/server/src/main/java/org/opensearch/action/ActionModule.java
+++ b/server/src/main/java/org/opensearch/action/ActionModule.java
@@ -107,8 +107,6 @@
 import org.opensearch.action.admin.cluster.snapshots.status.TransportSnapshotsStatusAction;
 import org.opensearch.action.admin.cluster.state.ClusterStateAction;
 import org.opensearch.action.admin.cluster.state.TransportClusterStateAction;
-import org.opensearch.action.admin.cluster.state.term.GetTermVersionAction;
-import org.opensearch.action.admin.cluster.state.term.TransportGetTermVersionAction;
 import org.opensearch.action.admin.cluster.stats.ClusterStatsAction;
 import org.opensearch.action.admin.cluster.stats.TransportClusterStatsAction;
 import org.opensearch.action.admin.cluster.storedscripts.DeleteStoredScriptAction;
@@ -283,6 +281,8 @@
 import org.opensearch.action.support.AutoCreateIndex;
 import org.opensearch.action.support.DestructiveOperations;
 import org.opensearch.action.support.TransportAction;
+import org.opensearch.action.support.clustermanager.term.GetTermVersionAction;
+import org.opensearch.action.support.clustermanager.term.TransportGetTermVersionAction;
 import org.opensearch.action.termvectors.MultiTermVectorsAction;
 import org.opensearch.action.termvectors.TermVectorsAction;
 import org.opensearch.action.termvectors.TransportMultiTermVectorsAction;
@@ -294,6 +294,7 @@
 import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
 import org.opensearch.cluster.node.DiscoveryNodes;
 import org.opensearch.common.NamedRegistry;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.inject.AbstractModule;
 import org.opensearch.common.inject.TypeLiteral;
 import org.opensearch.common.inject.multibindings.MapBinder;
@@ -1052,8 +1053,9 @@ public RestController getRestController() {
      * <p>
      * This class is modeled after {@link NamedRegistry} but provides both register and unregister capabilities.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "2.7.0")
     public static class DynamicActionRegistry {
         // This is the unmodifiable actions map created during node bootstrap, which
         // will continue to link ActionType and TransportAction pairs from core and plugin
diff --git a/server/src/main/java/org/opensearch/action/ActionRequest.java b/server/src/main/java/org/opensearch/action/ActionRequest.java
index 5313a05ad6fae..7ab87065bef7e 100644
--- a/server/src/main/java/org/opensearch/action/ActionRequest.java
+++ b/server/src/main/java/org/opensearch/action/ActionRequest.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.action;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.transport.TransportRequest;
@@ -43,6 +44,7 @@
  *
  * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public abstract class ActionRequest extends TransportRequest {
 
     public ActionRequest() {
diff --git a/server/src/main/java/org/opensearch/action/ActionType.java b/server/src/main/java/org/opensearch/action/ActionType.java
index dae931bdd1891..559dad73536e1 100644
--- a/server/src/main/java/org/opensearch/action/ActionType.java
+++ b/server/src/main/java/org/opensearch/action/ActionType.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.action;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.action.ActionResponse;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -43,6 +44,7 @@
  *
  * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class ActionType<Response extends ActionResponse> {
 
     private final String name;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java
index 8562a7eb37709..0917a0baff1ab 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodeStats.java
@@ -39,6 +39,7 @@
 import org.opensearch.cluster.routing.WeightedRoutingStats;
 import org.opensearch.cluster.service.ClusterManagerThrottlingStats;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.cache.service.NodeCacheStats;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.indices.breaker.AllCircuitBreakerStats;
@@ -158,6 +159,9 @@ public class NodeStats extends BaseNodeResponse implements ToXContentFragment {
     @Nullable
     private AdmissionControlStats admissionControlStats;
 
+    @Nullable
+    private NodeCacheStats nodeCacheStats;
+
     public NodeStats(StreamInput in) throws IOException {
         super(in);
         timestamp = in.readVLong();
@@ -234,6 +238,11 @@ public NodeStats(StreamInput in) throws IOException {
         } else {
             admissionControlStats = null;
         }
+        if (in.getVersion().onOrAfter(Version.V_2_14_0)) {
+            nodeCacheStats = in.readOptionalWriteable(NodeCacheStats::new);
+        } else {
+            nodeCacheStats = null;
+        }
     }
 
     public NodeStats(
@@ -264,7 +273,8 @@ public NodeStats(
         @Nullable SearchPipelineStats searchPipelineStats,
         @Nullable SegmentReplicationRejectionStats segmentReplicationRejectionStats,
         @Nullable RepositoriesStats repositoriesStats,
-        @Nullable AdmissionControlStats admissionControlStats
+        @Nullable AdmissionControlStats admissionControlStats,
+        @Nullable NodeCacheStats nodeCacheStats
     ) {
         super(node);
         this.timestamp = timestamp;
@@ -294,6 +304,7 @@ public NodeStats(
         this.segmentReplicationRejectionStats = segmentReplicationRejectionStats;
         this.repositoriesStats = repositoriesStats;
         this.admissionControlStats = admissionControlStats;
+        this.nodeCacheStats = nodeCacheStats;
     }
 
     public long getTimestamp() {
@@ -451,6 +462,11 @@ public AdmissionControlStats getAdmissionControlStats() {
         return admissionControlStats;
     }
 
+    @Nullable
+    public NodeCacheStats getNodeCacheStats() {
+        return nodeCacheStats;
+    }
+
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
@@ -506,6 +522,9 @@ public void writeTo(StreamOutput out) throws IOException {
         if (out.getVersion().onOrAfter(Version.V_2_12_0)) {
             out.writeOptionalWriteable(admissionControlStats);
         }
+        if (out.getVersion().onOrAfter(Version.V_2_14_0)) {
+            out.writeOptionalWriteable(nodeCacheStats);
+        }
     }
 
     @Override
@@ -609,6 +628,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
         if (getAdmissionControlStats() != null) {
             getAdmissionControlStats().toXContent(builder, params);
         }
+        if (getNodeCacheStats() != null) {
+            getNodeCacheStats().toXContent(builder, params);
+        }
         return builder;
     }
 }
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java
index 1af56f10b95ee..379836cf442e3 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/NodesStatsRequest.java
@@ -219,7 +219,8 @@ public enum Metric {
         RESOURCE_USAGE_STATS("resource_usage_stats"),
         SEGMENT_REPLICATION_BACKPRESSURE("segment_replication_backpressure"),
         REPOSITORIES("repositories"),
-        ADMISSION_CONTROL("admission_control");
+        ADMISSION_CONTROL("admission_control"),
+        CACHE_STATS("caches");
 
         private String metricName;
 
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java
index 1df73d3b4394d..2e93e5e7841cb 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/node/stats/TransportNodesStatsAction.java
@@ -128,7 +128,8 @@ protected NodeStats nodeOperation(NodeStatsRequest nodeStatsRequest) {
             NodesStatsRequest.Metric.RESOURCE_USAGE_STATS.containedIn(metrics),
             NodesStatsRequest.Metric.SEGMENT_REPLICATION_BACKPRESSURE.containedIn(metrics),
             NodesStatsRequest.Metric.REPOSITORIES.containedIn(metrics),
-            NodesStatsRequest.Metric.ADMISSION_CONTROL.containedIn(metrics)
+            NodesStatsRequest.Metric.ADMISSION_CONTROL.containedIn(metrics),
+            NodesStatsRequest.Metric.CACHE_STATS.containedIn(metrics)
         );
     }
 
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java
index 2f3cc77b05550..216e1fb2ed1cc 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java
@@ -42,9 +42,11 @@
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.block.ClusterBlockException;
 import org.opensearch.cluster.block.ClusterBlockLevel;
+import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
 import org.opensearch.cluster.metadata.Metadata;
 import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.cluster.node.DiscoveryNodes;
 import org.opensearch.cluster.routing.allocation.AllocationService;
 import org.opensearch.cluster.service.ClusterManagerTaskKeys;
 import org.opensearch.cluster.service.ClusterManagerTaskThrottler;
@@ -53,12 +55,21 @@
 import org.opensearch.common.Priority;
 import org.opensearch.common.inject.Inject;
 import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.settings.SettingsException;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.index.remote.RemoteMigrationIndexMetadataUpdater;
+import org.opensearch.node.remotestore.RemoteStoreNodeService;
 import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.transport.TransportService;
 
 import java.io.IOException;
+import java.util.Collection;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static org.opensearch.index.remote.RemoteMigrationIndexMetadataUpdater.indexHasAllRemoteStoreRelatedMetadata;
 
 /**
  * Transport action for updating cluster settings
@@ -251,6 +262,7 @@ public void onFailure(String source, Exception e) {
 
                 @Override
                 public ClusterState execute(final ClusterState currentState) {
+                    validateCompatibilityModeSettingRequest(request, state);
                     final ClusterState clusterState = updater.updateSettings(
                         currentState,
                         clusterSettings.upgradeSettings(request.transientSettings()),
@@ -264,4 +276,65 @@ public ClusterState execute(final ClusterState currentState) {
         );
     }
 
+    /**
+     * Runs various checks associated with changing cluster compatibility mode
+     * @param request cluster settings update request, for settings to be updated and new values
+     * @param clusterState current state of cluster, for information on nodes
+     */
+    public void validateCompatibilityModeSettingRequest(ClusterUpdateSettingsRequest request, ClusterState clusterState) {
+        Settings settings = Settings.builder().put(request.persistentSettings()).put(request.transientSettings()).build();
+        if (RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING.exists(settings)) {
+            String value = RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING.get(settings).mode;
+            validateAllNodesOfSameVersion(clusterState.nodes());
+            if (RemoteStoreNodeService.CompatibilityMode.STRICT.mode.equals(value)) {
+                validateAllNodesOfSameType(clusterState.nodes());
+                validateIndexSettings(clusterState);
+            }
+        }
+    }
+
+    /**
+     * Verifies that while trying to change the compatibility mode, all nodes must have the same version.
+     * If not, it throws SettingsException error
+     * @param discoveryNodes current discovery nodes in the cluster
+     */
+    private void validateAllNodesOfSameVersion(DiscoveryNodes discoveryNodes) {
+        if (discoveryNodes.getMaxNodeVersion().equals(discoveryNodes.getMinNodeVersion()) == false) {
+            throw new SettingsException("can not change the compatibility mode when all the nodes in cluster are not of the same version");
+        }
+    }
+
+    /**
+     * Verifies that while trying to switch to STRICT compatibility mode, all nodes must be of the
+     * same type (all remote or all non-remote). If not, it throws SettingsException error
+     * @param discoveryNodes current discovery nodes in the cluster
+     */
+    private void validateAllNodesOfSameType(DiscoveryNodes discoveryNodes) {
+        Set<Boolean> nodeTypes = discoveryNodes.getNodes()
+            .values()
+            .stream()
+            .map(DiscoveryNode::isRemoteStoreNode)
+            .collect(Collectors.toSet());
+        if (nodeTypes.size() != 1) {
+            throw new SettingsException(
+                "can not switch to STRICT compatibility mode when the cluster contains both remote and non-remote nodes"
+            );
+        }
+    }
+
+    /**
+     * Verifies that while trying to switch to STRICT compatibility mode,
+     * all indices in the cluster have {@link RemoteMigrationIndexMetadataUpdater#indexHasAllRemoteStoreRelatedMetadata(IndexMetadata)} as <code>true</code>.
+     * If not, throws {@link SettingsException}
+     * @param clusterState current cluster state
+     */
+    private void validateIndexSettings(ClusterState clusterState) {
+        Collection<IndexMetadata> allIndicesMetadata = clusterState.metadata().indices().values();
+        if (allIndicesMetadata.isEmpty() == false
+            && allIndicesMetadata.stream().anyMatch(indexMetadata -> indexHasAllRemoteStoreRelatedMetadata(indexMetadata) == false)) {
+            throw new SettingsException(
+                "can not switch to STRICT compatibility mode since all indices in the cluster does not have remote store based index settings"
+            );
+        }
+    }
 }
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java
index 7f6c039cf2ecc..4fc2acb2caa51 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java
@@ -356,11 +356,11 @@ private void loadRepositoryData(
                             state = SnapshotsInProgress.State.FAILED;
                             break;
                         case SUCCESS:
-                        case PARTIAL:
-                            // Translating both PARTIAL and SUCCESS to SUCCESS for now
-                            // TODO: add the differentiation on the metadata level in the next major release
                             state = SnapshotsInProgress.State.SUCCESS;
                             break;
+                        case PARTIAL:
+                            state = SnapshotsInProgress.State.PARTIAL;
+                            break;
                         default:
                             throw new IllegalArgumentException("Unknown snapshot state " + snapshotInfo.state());
                     }
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java
index 9c5dcc9e9de3f..e4f483f796f44 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java
+++ b/server/src/main/java/org/opensearch/action/admin/cluster/stats/TransportClusterStatsAction.java
@@ -172,6 +172,7 @@ protected ClusterStatsNodeResponse nodeOperation(ClusterStatsNodeRequest nodeReq
             false,
             false,
             false,
+            false,
             false
         );
         List<ShardStats> shardsStats = new ArrayList<>();
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java b/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java
index 86533f14e83e1..7edba143a72f0 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/GetFieldMappingsResponse.java
@@ -52,6 +52,7 @@
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
@@ -116,6 +117,11 @@ public class GetFieldMappingsResponse extends ActionResponse implements ToXConte
             String index = in.readString();
             if (in.getVersion().before(Version.V_2_0_0)) {
                 int typesSize = in.readVInt();
+                // if the requested field doesn't exist, type size in the received response from 1.x node is 0
+                if (typesSize == 0) {
+                    indexMapBuilder.put(index, Collections.emptyMap());
+                    continue;
+                }
                 if (typesSize != 1) {
                     throw new IllegalStateException("Expected single type but received [" + typesSize + "]");
                 }
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/shrink/TransportResizeAction.java b/server/src/main/java/org/opensearch/action/admin/indices/shrink/TransportResizeAction.java
index ca4c16935c2b9..cb41325c18a22 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/shrink/TransportResizeAction.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/shrink/TransportResizeAction.java
@@ -48,6 +48,7 @@
 import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.inject.Inject;
+import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -57,6 +58,9 @@
 import org.opensearch.index.IndexSettings;
 import org.opensearch.index.shard.DocsStats;
 import org.opensearch.index.store.StoreStats;
+import org.opensearch.node.remotestore.RemoteStoreNodeService;
+import org.opensearch.node.remotestore.RemoteStoreNodeService.CompatibilityMode;
+import org.opensearch.node.remotestore.RemoteStoreNodeService.Direction;
 import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.transport.TransportService;
 
@@ -67,6 +71,7 @@
 import java.util.function.IntFunction;
 
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
+import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED;
 
 /**
  * Main class to initiate resizing (shrink / split) an index into a new index
@@ -140,8 +145,8 @@ protected void clusterManagerOperation(
         // there is no need to fetch docs stats for split but we keep it simple and do it anyway for simplicity of the code
         final String sourceIndex = indexNameExpressionResolver.resolveDateMathExpression(resizeRequest.getSourceIndex());
         final String targetIndex = indexNameExpressionResolver.resolveDateMathExpression(resizeRequest.getTargetIndexRequest().index());
-
         IndexMetadata indexMetadata = state.metadata().index(sourceIndex);
+        ClusterSettings clusterSettings = clusterService.getClusterSettings();
         if (resizeRequest.getResizeType().equals(ResizeType.SHRINK)
             && state.metadata().isSegmentReplicationEnabled(sourceIndex)
             && indexMetadata != null
@@ -161,7 +166,7 @@ protected void clusterManagerOperation(
                             CreateIndexClusterStateUpdateRequest updateRequest = prepareCreateIndexRequest(resizeRequest, state, i -> {
                                 IndexShardStats shard = indicesStatsResponse.getIndex(sourceIndex).getIndexShards().get(i);
                                 return shard == null ? null : shard.getPrimary().getDocs();
-                            }, indicesStatsResponse.getPrimaries().store, sourceIndex, targetIndex);
+                            }, indicesStatsResponse.getPrimaries().store, clusterSettings, sourceIndex, targetIndex);
 
                             if (indicesStatsResponse.getIndex(sourceIndex)
                                 .getTotal()
@@ -200,7 +205,7 @@ protected void clusterManagerOperation(
                     CreateIndexClusterStateUpdateRequest updateRequest = prepareCreateIndexRequest(resizeRequest, state, i -> {
                         IndexShardStats shard = indicesStatsResponse.getIndex(sourceIndex).getIndexShards().get(i);
                         return shard == null ? null : shard.getPrimary().getDocs();
-                    }, indicesStatsResponse.getPrimaries().store, sourceIndex, targetIndex);
+                    }, indicesStatsResponse.getPrimaries().store, clusterSettings, sourceIndex, targetIndex);
                     createIndexService.createIndex(
                         updateRequest,
                         ActionListener.map(
@@ -223,6 +228,7 @@ static CreateIndexClusterStateUpdateRequest prepareCreateIndexRequest(
         final ClusterState state,
         final IntFunction<DocsStats> perShardDocStats,
         final StoreStats primaryShardsStoreStats,
+        final ClusterSettings clusterSettings,
         String sourceIndexName,
         String targetIndexName
     ) {
@@ -231,6 +237,7 @@ static CreateIndexClusterStateUpdateRequest prepareCreateIndexRequest(
         if (metadata == null) {
             throw new IndexNotFoundException(sourceIndexName);
         }
+        validateRemoteMigrationModeSettings(resizeRequest.getResizeType(), metadata, clusterSettings);
         final Settings.Builder targetIndexSettingsBuilder = Settings.builder()
             .put(targetIndex.settings())
             .normalizePrefix(IndexMetadata.INDEX_SETTING_PREFIX);
@@ -368,4 +375,39 @@ protected static int calculateTargetIndexShardsNum(
     protected String getClusterManagerActionName(DiscoveryNode node) {
         return super.getClusterManagerActionName(node);
     }
+
+    /**
+     * Reject resize request if cluster mode is [Mixed] and migration direction is [RemoteStore] and index is not on
+     * REMOTE_STORE_ENABLED node or [DocRep] and index is on REMOTE_STORE_ENABLED node.
+     * @param type resize type
+     * @param sourceIndexMetadata source index's metadata
+     * @param clusterSettings cluster settings
+     * @throws IllegalStateException if cluster mode is [Mixed] and migration direction is [RemoteStore] or [DocRep] and
+     *                               index's SETTING_REMOTE_STORE_ENABLED is not equal to the migration direction's value.
+     *                               For example, if migration direction is [RemoteStore] and index's SETTING_REMOTE_STORE_ENABLED
+     *                               is false, then throw IllegalStateException. If migration direction is [DocRep] and
+     *                               index's SETTING_REMOTE_STORE_ENABLED is true, then throw IllegalStateException.
+     */
+    private static void validateRemoteMigrationModeSettings(
+        final ResizeType type,
+        IndexMetadata sourceIndexMetadata,
+        ClusterSettings clusterSettings
+    ) {
+        CompatibilityMode compatibilityMode = clusterSettings.get(RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING);
+        if (compatibilityMode == CompatibilityMode.MIXED) {
+            boolean isRemoteStoreEnabled = sourceIndexMetadata.getSettings().getAsBoolean(SETTING_REMOTE_STORE_ENABLED, false);
+            Direction migrationDirection = clusterSettings.get(RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING);
+            boolean invalidConfiguration = (migrationDirection == Direction.REMOTE_STORE && isRemoteStoreEnabled == false)
+                || (migrationDirection == Direction.DOCREP && isRemoteStoreEnabled);
+            if (invalidConfiguration) {
+                throw new IllegalStateException(
+                    "Index "
+                        + type
+                        + " is not allowed as remote migration mode is mixed"
+                        + " and index is remote store "
+                        + (isRemoteStoreEnabled ? "enabled" : "disabled")
+                );
+            }
+        }
+    }
 }
diff --git a/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStatsFlags.java b/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStatsFlags.java
index a7d9f95b80f7b..4d108f8d78a69 100644
--- a/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStatsFlags.java
+++ b/server/src/main/java/org/opensearch/action/admin/indices/stats/CommonStatsFlags.java
@@ -34,6 +34,7 @@
 
 import org.opensearch.Version;
 import org.opensearch.common.annotation.PublicApi;
+import org.opensearch.common.cache.CacheType;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -42,6 +43,7 @@
 import java.io.IOException;
 import java.util.Collections;
 import java.util.EnumSet;
+import java.util.Set;
 
 /**
  * Common Stats Flags for OpenSearch
@@ -62,6 +64,9 @@ public class CommonStatsFlags implements Writeable, Cloneable {
     private boolean includeUnloadedSegments = false;
     private boolean includeAllShardIndexingPressureTrackers = false;
     private boolean includeOnlyTopIndexingPressureMetrics = false;
+    // Used for metric CACHE_STATS, to determine which caches to report stats for
+    private EnumSet<CacheType> includeCaches = EnumSet.noneOf(CacheType.class);
+    private String[] levels = new String[0];
 
     /**
      * @param flags flags to set. If no flags are supplied, default flags will be set.
@@ -91,6 +96,10 @@ public CommonStatsFlags(StreamInput in) throws IOException {
         includeUnloadedSegments = in.readBoolean();
         includeAllShardIndexingPressureTrackers = in.readBoolean();
         includeOnlyTopIndexingPressureMetrics = in.readBoolean();
+        if (in.getVersion().onOrAfter(Version.V_2_14_0)) {
+            includeCaches = in.readEnumSet(CacheType.class);
+            levels = in.readStringArray();
+        }
     }
 
     @Override
@@ -111,6 +120,10 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeBoolean(includeUnloadedSegments);
         out.writeBoolean(includeAllShardIndexingPressureTrackers);
         out.writeBoolean(includeOnlyTopIndexingPressureMetrics);
+        if (out.getVersion().onOrAfter(Version.V_2_14_0)) {
+            out.writeEnumSet(includeCaches);
+            out.writeStringArrayNullable(levels);
+        }
     }
 
     /**
@@ -125,6 +138,8 @@ public CommonStatsFlags all() {
         includeUnloadedSegments = false;
         includeAllShardIndexingPressureTrackers = false;
         includeOnlyTopIndexingPressureMetrics = false;
+        includeCaches = EnumSet.noneOf(CacheType.class);
+        levels = new String[0];
         return this;
     }
 
@@ -140,6 +155,8 @@ public CommonStatsFlags clear() {
         includeUnloadedSegments = false;
         includeAllShardIndexingPressureTrackers = false;
         includeOnlyTopIndexingPressureMetrics = false;
+        includeCaches = EnumSet.noneOf(CacheType.class);
+        levels = new String[0];
         return this;
     }
 
@@ -151,6 +168,14 @@ public Flag[] getFlags() {
         return flags.toArray(new Flag[0]);
     }
 
+    public Set<CacheType> getIncludeCaches() {
+        return includeCaches;
+    }
+
+    public String[] getLevels() {
+        return levels;
+    }
+
     /**
      * Sets specific search group stats to retrieve the stats for. Mainly affects search
      * when enabled.
@@ -206,6 +231,21 @@ public CommonStatsFlags includeOnlyTopIndexingPressureMetrics(boolean includeOnl
         return this;
     }
 
+    public CommonStatsFlags includeCacheType(CacheType cacheType) {
+        includeCaches.add(cacheType);
+        return this;
+    }
+
+    public CommonStatsFlags includeAllCacheTypes() {
+        includeCaches = EnumSet.allOf(CacheType.class);
+        return this;
+    }
+
+    public CommonStatsFlags setLevels(String[] inputLevels) {
+        levels = inputLevels;
+        return this;
+    }
+
     public boolean includeUnloadedSegments() {
         return this.includeUnloadedSegments;
     }
diff --git a/server/src/main/java/org/opensearch/action/bulk/BulkRequest.java b/server/src/main/java/org/opensearch/action/bulk/BulkRequest.java
index 47abd0337fcf9..7614206cd226f 100644
--- a/server/src/main/java/org/opensearch/action/bulk/BulkRequest.java
+++ b/server/src/main/java/org/opensearch/action/bulk/BulkRequest.java
@@ -34,6 +34,7 @@
 
 import org.apache.lucene.util.Accountable;
 import org.apache.lucene.util.RamUsageEstimator;
+import org.opensearch.Version;
 import org.opensearch.action.ActionRequest;
 import org.opensearch.action.ActionRequestValidationException;
 import org.opensearch.action.CompositeIndicesRequest;
@@ -80,7 +81,6 @@ public class BulkRequest extends ActionRequest implements CompositeIndicesReques
     private static final long SHALLOW_SIZE = RamUsageEstimator.shallowSizeOfInstance(BulkRequest.class);
 
     private static final int REQUEST_OVERHEAD = 50;
-
     /**
      * Requests that are part of this request. It is only possible to add things that are both {@link ActionRequest}s and
      * {@link WriteRequest}s to this but java doesn't support syntax to declare that everything in the array has both types so we declare
@@ -96,6 +96,7 @@ public class BulkRequest extends ActionRequest implements CompositeIndicesReques
     private String globalRouting;
     private String globalIndex;
     private Boolean globalRequireAlias;
+    private int batchSize = 1;
 
     private long sizeInBytes = 0;
 
@@ -107,6 +108,9 @@ public BulkRequest(StreamInput in) throws IOException {
         requests.addAll(in.readList(i -> DocWriteRequest.readDocumentRequest(null, i)));
         refreshPolicy = RefreshPolicy.readFrom(in);
         timeout = in.readTimeValue();
+        if (in.getVersion().onOrAfter(Version.V_2_14_0)) {
+            batchSize = in.readInt();
+        }
     }
 
     public BulkRequest(@Nullable String globalIndex) {
@@ -346,6 +350,27 @@ public final BulkRequest timeout(TimeValue timeout) {
         return this;
     }
 
+    /**
+     * Set batch size
+     * @param size batch size from input
+     * @return {@link BulkRequest}
+     */
+    public BulkRequest batchSize(int size) {
+        if (size < 1) {
+            throw new IllegalArgumentException("batch_size must be greater than 0");
+        }
+        this.batchSize = size;
+        return this;
+    }
+
+    /**
+     * Get batch size
+     * @return batch size
+     */
+    public int batchSize() {
+        return this.batchSize;
+    }
+
     /**
      * Note for internal callers (NOT high level rest client),
      * the global parameter setting is ignored when used with:
@@ -453,6 +478,9 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeCollection(requests, DocWriteRequest::writeDocumentRequest);
         refreshPolicy.writeTo(out);
         out.writeTimeValue(timeout);
+        if (out.getVersion().onOrAfter(Version.V_2_14_0)) {
+            out.writeInt(batchSize);
+        }
     }
 
     @Override
diff --git a/server/src/main/java/org/opensearch/action/bulk/BulkRequestParser.java b/server/src/main/java/org/opensearch/action/bulk/BulkRequestParser.java
index 3fadfe5f2cd6a..f705a218fb8e2 100644
--- a/server/src/main/java/org/opensearch/action/bulk/BulkRequestParser.java
+++ b/server/src/main/java/org/opensearch/action/bulk/BulkRequestParser.java
@@ -371,7 +371,7 @@ public void parse(
                         }
                         IndexRequest upsertRequest = updateRequest.upsertRequest();
                         if (upsertRequest != null) {
-                            upsertRequest.setPipeline(defaultPipeline);
+                            upsertRequest.setPipeline(pipeline);
                         }
 
                         updateRequestConsumer.accept(updateRequest);
diff --git a/server/src/main/java/org/opensearch/action/bulk/TransportBulkAction.java b/server/src/main/java/org/opensearch/action/bulk/TransportBulkAction.java
index 4a9b07c12821d..19ffb12859183 100644
--- a/server/src/main/java/org/opensearch/action/bulk/TransportBulkAction.java
+++ b/server/src/main/java/org/opensearch/action/bulk/TransportBulkAction.java
@@ -923,7 +923,8 @@ public boolean isForceExecution() {
                 }
             },
             bulkRequestModifier::markItemAsDropped,
-            executorName
+            executorName,
+            original
         );
     }
 
diff --git a/server/src/main/java/org/opensearch/action/bulk/TransportShardBulkAction.java b/server/src/main/java/org/opensearch/action/bulk/TransportShardBulkAction.java
index a7a13afd2597c..fdba8a42c0170 100644
--- a/server/src/main/java/org/opensearch/action/bulk/TransportShardBulkAction.java
+++ b/server/src/main/java/org/opensearch/action/bulk/TransportShardBulkAction.java
@@ -442,7 +442,7 @@ protected long primaryOperationSize(BulkShardRequest request) {
 
     @Override
     public ReplicationMode getReplicationMode(IndexShard indexShard) {
-        if (indexShard.isRemoteTranslogEnabled()) {
+        if (indexShard.indexSettings().isAssignedOnRemoteNode()) {
             return ReplicationMode.PRIMARY_TERM_VALIDATION;
         }
         return super.getReplicationMode(indexShard);
diff --git a/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java b/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java
index 0520a4a7aecec..9bf4a4b1e18f1 100644
--- a/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java
+++ b/server/src/main/java/org/opensearch/action/search/AbstractSearchAsyncAction.java
@@ -425,8 +425,10 @@ public final void executeNextPhase(SearchPhase currentPhase, SearchPhase nextPha
                             currentPhase.getName()
                         );
                     }
-                    onPhaseFailure(currentPhase, "Partial shards failure (" + discrepancy + " shards unavailable)", null);
-                    return;
+                    if (!request.indicesOptions().ignoreUnavailable()) {
+                        onPhaseFailure(currentPhase, "Partial shards failure (" + discrepancy + " shards unavailable)", null);
+                        return;
+                    }
                 }
             }
             if (logger.isTraceEnabled()) {
diff --git a/server/src/main/java/org/opensearch/action/search/SearchRequest.java b/server/src/main/java/org/opensearch/action/search/SearchRequest.java
index 3b8a6937815aa..4d3bb868b779a 100644
--- a/server/src/main/java/org/opensearch/action/search/SearchRequest.java
+++ b/server/src/main/java/org/opensearch/action/search/SearchRequest.java
@@ -40,6 +40,7 @@
 import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.common.Nullable;
 import org.opensearch.common.annotation.PublicApi;
+import org.opensearch.common.io.stream.BytesStreamOutput;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -161,6 +162,18 @@ public SearchRequest(String[] indices, SearchSourceBuilder source) {
         this.source = source;
     }
 
+    /**
+     * Deep clone a SearchRequest
+     *
+     * @return a copy of the current SearchRequest
+     */
+    public SearchRequest deepCopy() throws IOException {
+        BytesStreamOutput out = new BytesStreamOutput();
+        this.writeTo(out);
+        StreamInput in = out.bytes().streamInput();
+        return new SearchRequest(in);
+    }
+
     /**
      * Creates a new sub-search request starting from the original search request that is provided.
      * For internal use only, allows to fork a search request into multiple search requests that will be executed independently.
@@ -497,7 +510,7 @@ public PointInTimeBuilder pointInTimeBuilder() {
     }
 
     /**
-     * The tye of search to execute.
+     * The type of search to execute.
      */
     public SearchType searchType() {
         return searchType;
diff --git a/server/src/main/java/org/opensearch/action/search/SearchRequestBuilder.java b/server/src/main/java/org/opensearch/action/search/SearchRequestBuilder.java
index 9dac827e7d518..4a547ee2c82bd 100644
--- a/server/src/main/java/org/opensearch/action/search/SearchRequestBuilder.java
+++ b/server/src/main/java/org/opensearch/action/search/SearchRequestBuilder.java
@@ -363,6 +363,21 @@ public SearchRequestBuilder addScriptField(String name, Script script) {
         return this;
     }
 
+    /**
+     * Adds a derived field of a given type. The script provided will be used to derive the value
+     * of a given type. Thereafter, it can be treated as regular field of a given type to perform
+     * query on them.
+     *
+     * @param name   The name of the field to be used in various parts of the query. The name will also represent
+     *               the field value in the return hit.
+     * @param type   The type of derived field. All values emitted by script must be of this type
+     * @param script The script to use
+     */
+    public SearchRequestBuilder addDerivedField(String name, String type, Script script) {
+        sourceBuilder().derivedField(name, type, script);
+        return this;
+    }
+
     /**
      * Adds a sort against the given field name and the sort ordering.
      *
diff --git a/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java b/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java
index 65cfd35489033..143b01af3f62f 100644
--- a/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java
+++ b/server/src/main/java/org/opensearch/action/search/TransportSearchAction.java
@@ -457,7 +457,7 @@ private void executeRequest(
             PipelinedRequest searchRequest;
             ActionListener<SearchResponse> listener;
             try {
-                searchRequest = searchPipelineService.resolvePipeline(originalSearchRequest);
+                searchRequest = searchPipelineService.resolvePipeline(originalSearchRequest, indexNameExpressionResolver);
                 listener = searchRequest.transformResponseListener(updatedListener);
             } catch (Exception e) {
                 updatedListener.onFailure(e);
diff --git a/server/src/main/java/org/opensearch/action/support/AutoCreateIndex.java b/server/src/main/java/org/opensearch/action/support/AutoCreateIndex.java
index 9e8cbd7bf40c2..ce0a8f84d066f 100644
--- a/server/src/main/java/org/opensearch/action/support/AutoCreateIndex.java
+++ b/server/src/main/java/org/opensearch/action/support/AutoCreateIndex.java
@@ -43,7 +43,6 @@
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.common.Strings;
 import org.opensearch.index.IndexNotFoundException;
-import org.opensearch.index.mapper.MapperService;
 import org.opensearch.indices.SystemIndices;
 
 import java.util.ArrayList;
@@ -64,8 +63,6 @@ public final class AutoCreateIndex {
         Property.NodeScope,
         Setting.Property.Dynamic
     );
-
-    private final boolean dynamicMappingDisabled;
     private final IndexNameExpressionResolver resolver;
     private final SystemIndices systemIndices;
     private volatile AutoCreate autoCreate;
@@ -77,7 +74,6 @@ public AutoCreateIndex(
         SystemIndices systemIndices
     ) {
         this.resolver = resolver;
-        dynamicMappingDisabled = !MapperService.INDEX_MAPPER_DYNAMIC_SETTING.get(settings);
         this.systemIndices = systemIndices;
         this.autoCreate = AUTO_CREATE_INDEX_SETTING.get(settings);
         clusterSettings.addSettingsUpdateConsumer(AUTO_CREATE_INDEX_SETTING, this::setAutoCreate);
@@ -109,9 +105,6 @@ public boolean shouldAutoCreate(String index, ClusterState state) {
         if (autoCreate.autoCreateIndex == false) {
             throw new IndexNotFoundException("[" + AUTO_CREATE_INDEX_SETTING.getKey() + "] is [false]", index);
         }
-        if (dynamicMappingDisabled) {
-            throw new IndexNotFoundException("[" + MapperService.INDEX_MAPPER_DYNAMIC_SETTING.getKey() + "] is [false]", index);
-        }
         // matches not set, default value of "true"
         if (autoCreate.expressions.isEmpty()) {
             return true;
diff --git a/server/src/main/java/org/opensearch/action/support/TransportAction.java b/server/src/main/java/org/opensearch/action/support/TransportAction.java
index 72aae210d61ae..f71347f6f1d07 100644
--- a/server/src/main/java/org/opensearch/action/support/TransportAction.java
+++ b/server/src/main/java/org/opensearch/action/support/TransportAction.java
@@ -36,6 +36,7 @@
 import org.apache.logging.log4j.Logger;
 import org.opensearch.action.ActionRequest;
 import org.opensearch.action.ActionRequestValidationException;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.lease.Releasable;
 import org.opensearch.common.lease.Releasables;
 import org.opensearch.common.util.concurrent.ThreadContext;
@@ -52,8 +53,9 @@
 /**
  * Base class for a transport action
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public abstract class TransportAction<Request extends ActionRequest, Response extends ActionResponse> {
 
     public final String actionName;
diff --git a/server/src/main/java/org/opensearch/action/support/clustermanager/ClusterManagerNodeRequest.java b/server/src/main/java/org/opensearch/action/support/clustermanager/ClusterManagerNodeRequest.java
index a43d6fb0b1e7a..03fc41e829e3d 100644
--- a/server/src/main/java/org/opensearch/action/support/clustermanager/ClusterManagerNodeRequest.java
+++ b/server/src/main/java/org/opensearch/action/support/clustermanager/ClusterManagerNodeRequest.java
@@ -33,6 +33,7 @@
 package org.opensearch.action.support.clustermanager;
 
 import org.opensearch.action.ActionRequest;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -42,8 +43,9 @@
 /**
  * A based request for cluster-manager based operation.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public abstract class ClusterManagerNodeRequest<Request extends ClusterManagerNodeRequest<Request>> extends ActionRequest {
 
     public static final TimeValue DEFAULT_CLUSTER_MANAGER_NODE_TIMEOUT = TimeValue.timeValueSeconds(30);
diff --git a/server/src/main/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeAction.java b/server/src/main/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeAction.java
index 5f57658e33924..080b0d607e991 100644
--- a/server/src/main/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeAction.java
+++ b/server/src/main/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeAction.java
@@ -37,13 +37,13 @@
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.opensearch.action.ActionListenerResponseHandler;
 import org.opensearch.action.ActionRunnable;
-import org.opensearch.action.admin.cluster.state.term.GetTermVersionAction;
-import org.opensearch.action.admin.cluster.state.term.GetTermVersionRequest;
-import org.opensearch.action.admin.cluster.state.term.GetTermVersionResponse;
 import org.opensearch.action.bulk.BackoffPolicy;
 import org.opensearch.action.support.ActionFilters;
 import org.opensearch.action.support.HandledTransportAction;
 import org.opensearch.action.support.RetryableAction;
+import org.opensearch.action.support.clustermanager.term.GetTermVersionAction;
+import org.opensearch.action.support.clustermanager.term.GetTermVersionRequest;
+import org.opensearch.action.support.clustermanager.term.GetTermVersionResponse;
 import org.opensearch.cluster.ClusterManagerNodeChangePredicate;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.ClusterStateObserver;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/state/term/GetTermVersionAction.java b/server/src/main/java/org/opensearch/action/support/clustermanager/term/GetTermVersionAction.java
similarity index 83%
rename from server/src/main/java/org/opensearch/action/admin/cluster/state/term/GetTermVersionAction.java
rename to server/src/main/java/org/opensearch/action/support/clustermanager/term/GetTermVersionAction.java
index 3344fd549b23f..2401dddd0cab3 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/state/term/GetTermVersionAction.java
+++ b/server/src/main/java/org/opensearch/action/support/clustermanager/term/GetTermVersionAction.java
@@ -6,7 +6,7 @@
  * compatible open source license.
  */
 
-package org.opensearch.action.admin.cluster.state.term;
+package org.opensearch.action.support.clustermanager.term;
 
 import org.opensearch.action.ActionType;
 
@@ -18,7 +18,7 @@
 public class GetTermVersionAction extends ActionType<GetTermVersionResponse> {
 
     public static final GetTermVersionAction INSTANCE = new GetTermVersionAction();
-    public static final String NAME = "cluster:monitor/term";
+    public static final String NAME = "internal:monitor/term";
 
     private GetTermVersionAction() {
         super(NAME, GetTermVersionResponse::new);
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/state/term/GetTermVersionRequest.java b/server/src/main/java/org/opensearch/action/support/clustermanager/term/GetTermVersionRequest.java
similarity index 93%
rename from server/src/main/java/org/opensearch/action/admin/cluster/state/term/GetTermVersionRequest.java
rename to server/src/main/java/org/opensearch/action/support/clustermanager/term/GetTermVersionRequest.java
index b099f8087bd15..507997a1f7e7a 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/state/term/GetTermVersionRequest.java
+++ b/server/src/main/java/org/opensearch/action/support/clustermanager/term/GetTermVersionRequest.java
@@ -6,7 +6,7 @@
  * compatible open source license.
  */
 
-package org.opensearch.action.admin.cluster.state.term;
+package org.opensearch.action.support.clustermanager.term;
 
 import org.opensearch.action.ActionRequestValidationException;
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeReadRequest;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/state/term/GetTermVersionResponse.java b/server/src/main/java/org/opensearch/action/support/clustermanager/term/GetTermVersionResponse.java
similarity index 96%
rename from server/src/main/java/org/opensearch/action/admin/cluster/state/term/GetTermVersionResponse.java
rename to server/src/main/java/org/opensearch/action/support/clustermanager/term/GetTermVersionResponse.java
index 16b355a80d1f2..0906abe57d547 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/state/term/GetTermVersionResponse.java
+++ b/server/src/main/java/org/opensearch/action/support/clustermanager/term/GetTermVersionResponse.java
@@ -6,7 +6,7 @@
  * compatible open source license.
  */
 
-package org.opensearch.action.admin.cluster.state.term;
+package org.opensearch.action.support.clustermanager.term;
 
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.coordination.ClusterStateTermVersion;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/state/term/TransportGetTermVersionAction.java b/server/src/main/java/org/opensearch/action/support/clustermanager/term/TransportGetTermVersionAction.java
similarity index 97%
rename from server/src/main/java/org/opensearch/action/admin/cluster/state/term/TransportGetTermVersionAction.java
rename to server/src/main/java/org/opensearch/action/support/clustermanager/term/TransportGetTermVersionAction.java
index 88305252aa99c..4752a99c910e4 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/state/term/TransportGetTermVersionAction.java
+++ b/server/src/main/java/org/opensearch/action/support/clustermanager/term/TransportGetTermVersionAction.java
@@ -6,7 +6,7 @@
  * compatible open source license.
  */
 
-package org.opensearch.action.admin.cluster.state.term;
+package org.opensearch.action.support.clustermanager.term;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/state/term/package-info.java b/server/src/main/java/org/opensearch/action/support/clustermanager/term/package-info.java
similarity index 80%
rename from server/src/main/java/org/opensearch/action/admin/cluster/state/term/package-info.java
rename to server/src/main/java/org/opensearch/action/support/clustermanager/term/package-info.java
index 0ee559c527d7d..229c405df2d7c 100644
--- a/server/src/main/java/org/opensearch/action/admin/cluster/state/term/package-info.java
+++ b/server/src/main/java/org/opensearch/action/support/clustermanager/term/package-info.java
@@ -7,4 +7,4 @@
  */
 
 /** Cluster Term transport handler. */
-package org.opensearch.action.admin.cluster.state.term;
+package org.opensearch.action.support.clustermanager.term;
diff --git a/server/src/main/java/org/opensearch/action/support/replication/ReplicationModeAwareProxy.java b/server/src/main/java/org/opensearch/action/support/replication/ReplicationModeAwareProxy.java
index 189bc82348a0c..9f5e31a9c6926 100644
--- a/server/src/main/java/org/opensearch/action/support/replication/ReplicationModeAwareProxy.java
+++ b/server/src/main/java/org/opensearch/action/support/replication/ReplicationModeAwareProxy.java
@@ -9,6 +9,8 @@
 package org.opensearch.action.support.replication;
 
 import org.opensearch.action.support.replication.ReplicationOperation.ReplicaResponse;
+import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.cluster.node.DiscoveryNodes;
 import org.opensearch.cluster.routing.ShardRouting;
 import org.opensearch.core.action.ActionListener;
 
@@ -31,14 +33,22 @@ public class ReplicationModeAwareProxy<ReplicaRequest extends ReplicationRequest
      */
     private final ReplicationOperation.Replicas<ReplicaRequest> primaryTermValidationProxy;
 
+    private final DiscoveryNodes discoveryNodes;
+
+    private final boolean isRemoteEnabled;
+
     public ReplicationModeAwareProxy(
         ReplicationMode replicationModeOverride,
+        DiscoveryNodes discoveryNodes,
         ReplicationOperation.Replicas<ReplicaRequest> replicasProxy,
-        ReplicationOperation.Replicas<ReplicaRequest> primaryTermValidationProxy
+        ReplicationOperation.Replicas<ReplicaRequest> primaryTermValidationProxy,
+        boolean remoteIndexSettingsEnabled
     ) {
         super(replicasProxy);
         this.replicationModeOverride = Objects.requireNonNull(replicationModeOverride);
         this.primaryTermValidationProxy = Objects.requireNonNull(primaryTermValidationProxy);
+        this.discoveryNodes = discoveryNodes;
+        this.isRemoteEnabled = remoteIndexSettingsEnabled;
     }
 
     @Override
@@ -60,16 +70,26 @@ protected void performOnReplicaProxy(
 
     @Override
     ReplicationMode determineReplicationMode(ShardRouting shardRouting, ShardRouting primaryRouting) {
-
         // If the current routing is the primary, then it does not need to be replicated
         if (shardRouting.isSameAllocation(primaryRouting)) {
             return ReplicationMode.NO_REPLICATION;
         }
-
+        // Perform full replication during primary relocation
         if (primaryRouting.relocating() && shardRouting.isSameAllocation(primaryRouting.getTargetRelocatingShard())) {
             return ReplicationMode.FULL_REPLICATION;
         }
-
+        /*
+        Only applicable during remote store migration.
+        During the migration process, remote based index settings will not be enabled,
+        thus we will rely on node attributes to figure out the replication mode
+         */
+        if (isRemoteEnabled == false) {
+            DiscoveryNode targetNode = discoveryNodes.get(shardRouting.currentNodeId());
+            if (targetNode != null && targetNode.isRemoteStoreNode() == false) {
+                // Perform full replication if replica is hosted on a non-remote node.
+                return ReplicationMode.FULL_REPLICATION;
+            }
+        }
         return replicationModeOverride;
     }
 }
diff --git a/server/src/main/java/org/opensearch/action/support/replication/TransportReplicationAction.java b/server/src/main/java/org/opensearch/action/support/replication/TransportReplicationAction.java
index 95f998e2d89c2..49a96603f6802 100644
--- a/server/src/main/java/org/opensearch/action/support/replication/TransportReplicationAction.java
+++ b/server/src/main/java/org/opensearch/action/support/replication/TransportReplicationAction.java
@@ -356,7 +356,7 @@ public void performOn(
      * @return the overridden replication mode.
      */
     public ReplicationMode getReplicationMode(IndexShard indexShard) {
-        if (indexShard.isRemoteTranslogEnabled()) {
+        if (indexShard.indexSettings().isAssignedOnRemoteNode()) {
             return ReplicationMode.NO_REPLICATION;
         }
         return ReplicationMode.FULL_REPLICATION;
@@ -642,8 +642,14 @@ public void handleException(TransportException exp) {
                         primaryRequest.getPrimaryTerm(),
                         initialRetryBackoffBound,
                         retryTimeout,
-                        indexShard.isRemoteTranslogEnabled()
-                            ? new ReplicationModeAwareProxy<>(getReplicationMode(indexShard), replicasProxy, termValidationProxy)
+                        indexShard.indexSettings().isAssignedOnRemoteNode()
+                            ? new ReplicationModeAwareProxy<>(
+                                getReplicationMode(indexShard),
+                                clusterState.getNodes(),
+                                replicasProxy,
+                                termValidationProxy,
+                                indexShard.isRemoteTranslogEnabled()
+                            )
                             : new FanoutReplicationProxy<>(replicasProxy)
                     ).execute();
                 }
diff --git a/server/src/main/java/org/opensearch/client/node/NodeClient.java b/server/src/main/java/org/opensearch/client/node/NodeClient.java
index 6e1bb6ce79349..5780e4c1e648a 100644
--- a/server/src/main/java/org/opensearch/client/node/NodeClient.java
+++ b/server/src/main/java/org/opensearch/client/node/NodeClient.java
@@ -39,6 +39,7 @@
 import org.opensearch.client.Client;
 import org.opensearch.client.support.AbstractClient;
 import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.action.ActionResponse;
@@ -53,8 +54,9 @@
 /**
  * Client that executes actions on the local node.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class NodeClient extends AbstractClient {
 
     private DynamicActionRegistry actionRegistry;
diff --git a/server/src/main/java/org/opensearch/cluster/ClusterManagerMetrics.java b/server/src/main/java/org/opensearch/cluster/ClusterManagerMetrics.java
new file mode 100644
index 0000000000000..5877398004f93
--- /dev/null
+++ b/server/src/main/java/org/opensearch/cluster/ClusterManagerMetrics.java
@@ -0,0 +1,70 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.cluster;
+
+import org.opensearch.telemetry.metrics.Histogram;
+import org.opensearch.telemetry.metrics.MetricsRegistry;
+import org.opensearch.telemetry.metrics.tags.Tags;
+
+import java.util.Objects;
+import java.util.Optional;
+
+/**
+ * Class containing metrics (counters/latency) specific to ClusterManager.
+ */
+public final class ClusterManagerMetrics {
+
+    private static final String LATENCY_METRIC_UNIT_MS = "ms";
+
+    public final Histogram clusterStateAppliersHistogram;
+    public final Histogram clusterStateListenersHistogram;
+    public final Histogram rerouteHistogram;
+    public final Histogram clusterStateComputeHistogram;
+    public final Histogram clusterStatePublishHistogram;
+
+    public ClusterManagerMetrics(MetricsRegistry metricsRegistry) {
+        clusterStateAppliersHistogram = metricsRegistry.createHistogram(
+            "cluster.state.appliers.latency",
+            "Histogram for tracking the latency of cluster state appliers",
+            LATENCY_METRIC_UNIT_MS
+        );
+        clusterStateListenersHistogram = metricsRegistry.createHistogram(
+            "cluster.state.listeners.latency",
+            "Histogram for tracking the latency of cluster state listeners",
+            LATENCY_METRIC_UNIT_MS
+        );
+        rerouteHistogram = metricsRegistry.createHistogram(
+            "allocation.reroute.latency",
+            "Histogram for recording latency of shard re-routing",
+            LATENCY_METRIC_UNIT_MS
+        );
+        clusterStateComputeHistogram = metricsRegistry.createHistogram(
+            "cluster.state.new.compute.latency",
+            "Histogram for recording time taken to compute new cluster state",
+            LATENCY_METRIC_UNIT_MS
+        );
+        clusterStatePublishHistogram = metricsRegistry.createHistogram(
+            "cluster.state.publish.success.latency",
+            "Histogram for recording time taken to publish a new cluster state",
+            LATENCY_METRIC_UNIT_MS
+        );
+    }
+
+    public static void recordLatency(Histogram histogram, Double value) {
+        histogram.record(value);
+    }
+
+    public static void recordLatency(Histogram histogram, Double value, Optional<Tags> tags) {
+        if (Objects.isNull(tags) || tags.isEmpty()) {
+            histogram.record(value);
+            return;
+        }
+        histogram.record(value, tags.get());
+    }
+}
diff --git a/server/src/main/java/org/opensearch/cluster/ClusterModule.java b/server/src/main/java/org/opensearch/cluster/ClusterModule.java
index c8eeac94bff39..f56c906db1002 100644
--- a/server/src/main/java/org/opensearch/cluster/ClusterModule.java
+++ b/server/src/main/java/org/opensearch/cluster/ClusterModule.java
@@ -93,6 +93,7 @@
 import org.opensearch.core.common.io.stream.Writeable.Reader;
 import org.opensearch.core.xcontent.NamedXContentRegistry;
 import org.opensearch.gateway.GatewayAllocator;
+import org.opensearch.gateway.ShardsBatchGatewayAllocator;
 import org.opensearch.ingest.IngestMetadata;
 import org.opensearch.persistent.PersistentTasksCustomMetadata;
 import org.opensearch.persistent.PersistentTasksNodeService;
@@ -102,7 +103,6 @@
 import org.opensearch.snapshots.SnapshotsInfoService;
 import org.opensearch.tasks.Task;
 import org.opensearch.tasks.TaskResultsService;
-import org.opensearch.telemetry.metrics.MetricsRegistry;
 
 import java.util.ArrayList;
 import java.util.Collection;
@@ -147,7 +147,7 @@ public ClusterModule(
         ClusterInfoService clusterInfoService,
         SnapshotsInfoService snapshotsInfoService,
         ThreadContext threadContext,
-        MetricsRegistry metricsRegistry
+        ClusterManagerMetrics clusterManagerMetrics
     ) {
         this.clusterPlugins = clusterPlugins;
         this.deciderList = createAllocationDeciders(settings, clusterService.getClusterSettings(), clusterPlugins);
@@ -160,7 +160,8 @@ public ClusterModule(
             shardsAllocator,
             clusterInfoService,
             snapshotsInfoService,
-            metricsRegistry
+            settings,
+            clusterManagerMetrics
         );
     }
 
@@ -431,6 +432,7 @@ public AllocationService getAllocationService() {
     @Override
     protected void configure() {
         bind(GatewayAllocator.class).asEagerSingleton();
+        bind(ShardsBatchGatewayAllocator.class).asEagerSingleton();
         bind(AllocationService.class).toInstance(allocationService);
         bind(ClusterService.class).toInstance(clusterService);
         bind(NodeConnectionsService.class).asEagerSingleton();
@@ -450,10 +452,10 @@ protected void configure() {
         bind(ShardsAllocator.class).toInstance(shardsAllocator);
     }
 
-    public void setExistingShardsAllocators(GatewayAllocator gatewayAllocator) {
+    public void setExistingShardsAllocators(GatewayAllocator gatewayAllocator, ShardsBatchGatewayAllocator shardsBatchGatewayAllocator) {
         final Map<String, ExistingShardsAllocator> existingShardsAllocators = new HashMap<>();
         existingShardsAllocators.put(GatewayAllocator.ALLOCATOR_NAME, gatewayAllocator);
-
+        existingShardsAllocators.put(ShardsBatchGatewayAllocator.ALLOCATOR_NAME, shardsBatchGatewayAllocator);
         for (ClusterPlugin clusterPlugin : clusterPlugins) {
             for (Map.Entry<String, ExistingShardsAllocator> existingShardsAllocatorEntry : clusterPlugin.getExistingShardsAllocators()
                 .entrySet()) {
diff --git a/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java b/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java
index 3de23d2490c63..d658f38430dd9 100644
--- a/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java
+++ b/server/src/main/java/org/opensearch/cluster/SnapshotsInProgress.java
@@ -747,7 +747,12 @@ public void writeTo(StreamOutput out) throws IOException {
             snapshot.writeTo(out);
             out.writeBoolean(includeGlobalState);
             out.writeBoolean(partial);
-            out.writeByte(state.value());
+            if ((out.getVersion().before(Version.V_2_14_0)) && state == State.PARTIAL) {
+                // Setting to SUCCESS for partial snapshots in older versions to maintain backward compatibility
+                out.writeByte(State.SUCCESS.value());
+            } else {
+                out.writeByte(state.value());
+            }
             out.writeList(indices);
             out.writeLong(startTime);
             out.writeMap(shards, (o, v) -> v.writeTo(o), (o, v) -> v.writeTo(o));
@@ -937,7 +942,8 @@ public enum State {
         STARTED((byte) 1, false),
         SUCCESS((byte) 2, true),
         FAILED((byte) 3, true),
-        ABORTED((byte) 4, false);
+        ABORTED((byte) 4, false),
+        PARTIAL((byte) 5, false);
 
         private final byte value;
 
@@ -968,6 +974,8 @@ public static State fromValue(byte value) {
                     return FAILED;
                 case 4:
                     return ABORTED;
+                case 5:
+                    return PARTIAL;
                 default:
                     throw new IllegalArgumentException("No snapshot state for value [" + value + "]");
             }
diff --git a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java
index 5d896e392e6bc..5475470b81b93 100644
--- a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java
+++ b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java
@@ -31,6 +31,7 @@
 
 package org.opensearch.cluster.coordination;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.opensearch.Version;
 import org.opensearch.cluster.ClusterState;
@@ -57,6 +58,7 @@
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
@@ -66,6 +68,7 @@
 import static org.opensearch.cluster.decommission.DecommissionHelper.nodeCommissioned;
 import static org.opensearch.gateway.GatewayService.STATE_NOT_RECOVERED_BLOCK;
 import static org.opensearch.node.remotestore.RemoteStoreNodeService.CompatibilityMode;
+import static org.opensearch.node.remotestore.RemoteStoreNodeService.CompatibilityMode.MIXED;
 import static org.opensearch.node.remotestore.RemoteStoreNodeService.CompatibilityMode.STRICT;
 import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING;
 
@@ -78,7 +81,7 @@ public class JoinTaskExecutor implements ClusterStateTaskExecutor<JoinTaskExecut
 
     private final AllocationService allocationService;
 
-    private final Logger logger;
+    private static Logger logger = LogManager.getLogger(JoinTaskExecutor.class);
     private final RerouteService rerouteService;
 
     private final RemoteStoreNodeService remoteStoreNodeService;
@@ -142,7 +145,7 @@ public JoinTaskExecutor(
         RemoteStoreNodeService remoteStoreNodeService
     ) {
         this.allocationService = allocationService;
-        this.logger = logger;
+        JoinTaskExecutor.logger = logger;
         this.rerouteService = rerouteService;
         this.remoteStoreNodeService = remoteStoreNodeService;
     }
@@ -521,7 +524,18 @@ private static void ensureRemoteStoreNodesCompatibility(DiscoveryNode joiningNod
                 }
             }
         } else {
-            if (remoteStoreCompatibilityMode == CompatibilityMode.MIXED) {
+            if (MIXED.equals(remoteStoreCompatibilityMode)) {
+                if (joiningNode.getVersion().after(currentNodes.getMaxNodeVersion())) {
+                    String reason = String.format(
+                        Locale.ROOT,
+                        "remote migration : a node [%s] of higher version [%s] is not allowed to join a cluster with maximum version [%s]",
+                        joiningNode,
+                        joiningNode.getVersion(),
+                        currentNodes.getMaxNodeVersion()
+                    );
+                    logger.warn(reason);
+                    throw new IllegalStateException(reason);
+                }
                 if (joiningNode.isRemoteStoreNode()) {
                     Optional<DiscoveryNode> remoteDN = existingNodes.stream().filter(DiscoveryNode::isRemoteStoreNode).findFirst();
                     remoteDN.ifPresent(discoveryNode -> ensureRemoteStoreNodesCompatibility(joiningNode, discoveryNode));
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/IndexNameExpressionResolver.java b/server/src/main/java/org/opensearch/cluster/metadata/IndexNameExpressionResolver.java
index 9a3b569a7ac3d..24ff83d638d4b 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/IndexNameExpressionResolver.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/IndexNameExpressionResolver.java
@@ -380,7 +380,13 @@ private void checkSystemIndexAccess(Context context, Metadata metadata, Set<Inde
     private static boolean shouldTrackConcreteIndex(Context context, IndicesOptions options, IndexMetadata index) {
         if (index.getState() == IndexMetadata.State.CLOSE) {
             if (options.forbidClosedIndices() && options.ignoreUnavailable() == false) {
-                throw new IndexClosedException(index.getIndex());
+                if (options.expandWildcardsClosed() == true && options.getExpandWildcards().size() == 1) {
+                    throw new IllegalArgumentException(
+                        "To expand [" + index.getState() + "] wildcard, please set forbid_closed_indices to `false`"
+                    );
+                } else {
+                    throw new IndexClosedException(index.getIndex());
+                }
             } else {
                 return options.forbidClosedIndices() == false && addIndex(index, context);
             }
diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java
index f6a14d8ec9d63..121f8d935cf48 100644
--- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java
+++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java
@@ -52,6 +52,7 @@
 import org.opensearch.cluster.block.ClusterBlock;
 import org.opensearch.cluster.block.ClusterBlockLevel;
 import org.opensearch.cluster.block.ClusterBlocks;
+import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.cluster.node.DiscoveryNodes;
 import org.opensearch.cluster.routing.IndexRoutingTable;
 import org.opensearch.cluster.routing.RoutingTable;
@@ -88,18 +89,21 @@
 import org.opensearch.index.mapper.MapperService;
 import org.opensearch.index.mapper.MapperService.MergeReason;
 import org.opensearch.index.query.QueryShardContext;
-import org.opensearch.index.remote.RemoteStorePathResolver;
-import org.opensearch.index.remote.RemoteStorePathType;
+import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm;
+import org.opensearch.index.remote.RemoteStoreEnums.PathType;
+import org.opensearch.index.remote.RemoteStorePathStrategy;
+import org.opensearch.index.remote.RemoteStorePathStrategyResolver;
 import org.opensearch.index.shard.IndexSettingProvider;
 import org.opensearch.index.translog.Translog;
 import org.opensearch.indices.IndexCreationException;
 import org.opensearch.indices.IndicesService;
 import org.opensearch.indices.InvalidIndexNameException;
+import org.opensearch.indices.RemoteStoreSettings;
 import org.opensearch.indices.ShardLimitValidator;
 import org.opensearch.indices.SystemIndices;
 import org.opensearch.indices.replication.common.ReplicationType;
-import org.opensearch.node.Node;
 import org.opensearch.node.remotestore.RemoteStoreNodeAttribute;
+import org.opensearch.node.remotestore.RemoteStoreNodeService;
 import org.opensearch.threadpool.ThreadPool;
 
 import java.io.IOException;
@@ -113,6 +117,7 @@
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicInteger;
@@ -141,6 +146,8 @@
 import static org.opensearch.index.IndexModule.INDEX_STORE_TYPE_SETTING;
 import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING;
 import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteDataAttributePresent;
+import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING;
+import static org.opensearch.node.remotestore.RemoteStoreNodeService.isMigratingToRemoteStore;
 
 /**
  * Service responsible for submitting create index requests
@@ -170,7 +177,7 @@ public class MetadataCreateIndexService {
     private AwarenessReplicaBalance awarenessReplicaBalance;
 
     @Nullable
-    private final RemoteStorePathResolver remoteStorePathResolver;
+    private final RemoteStorePathStrategyResolver remoteStorePathStrategyResolver;
 
     public MetadataCreateIndexService(
         final Settings settings,
@@ -185,7 +192,8 @@ public MetadataCreateIndexService(
         final NamedXContentRegistry xContentRegistry,
         final SystemIndices systemIndices,
         final boolean forbidPrivateIndexSettings,
-        final AwarenessReplicaBalance awarenessReplicaBalance
+        final AwarenessReplicaBalance awarenessReplicaBalance,
+        final RemoteStoreSettings remoteStoreSettings
     ) {
         this.settings = settings;
         this.clusterService = clusterService;
@@ -203,8 +211,9 @@ public MetadataCreateIndexService(
 
         // Task is onboarded for throttling, it will get retried from associated TransportClusterManagerNodeAction.
         createIndexTaskKey = clusterService.registerClusterManagerTask(ClusterManagerTaskKeys.CREATE_INDEX_KEY, true);
-        remoteStorePathResolver = isRemoteDataAttributePresent(settings)
-            ? new RemoteStorePathResolver(clusterService.getClusterSettings())
+        Supplier<Version> minNodeVersionSupplier = () -> clusterService.state().nodes().getMinNodeVersion();
+        remoteStorePathStrategyResolver = isRemoteDataAttributePresent(settings)
+            ? new RemoteStorePathStrategyResolver(remoteStoreSettings, minNodeVersionSupplier)
             : null;
     }
 
@@ -553,7 +562,7 @@ IndexMetadata buildAndValidateTemporaryIndexMetadata(
         tmpImdBuilder.setRoutingNumShards(routingNumShards);
         tmpImdBuilder.settings(indexSettings);
         tmpImdBuilder.system(isSystem);
-        addRemoteCustomData(tmpImdBuilder);
+        addRemoteStorePathStrategyInCustomData(tmpImdBuilder, true);
 
         // Set up everything, now locally create the index to see that things are ok, and apply
         IndexMetadata tempMetadata = tmpImdBuilder.build();
@@ -562,20 +571,30 @@ IndexMetadata buildAndValidateTemporaryIndexMetadata(
         return tempMetadata;
     }
 
-    public void addRemoteCustomData(IndexMetadata.Builder tmpImdBuilder) {
-        if (remoteStorePathResolver != null) {
-            // It is possible that remote custom data exists already. In such cases, we need to only update the path type
-            // in the remote store custom data map.
-            Map<String, String> existingRemoteCustomData = tmpImdBuilder.removeCustom(IndexMetadata.REMOTE_STORE_CUSTOM_KEY);
-            Map<String, String> remoteCustomData = existingRemoteCustomData == null
-                ? new HashMap<>()
-                : new HashMap<>(existingRemoteCustomData);
-            // Determine the path type for use using the remoteStorePathResolver.
-            String newPathType = remoteStorePathResolver.resolveType().toString();
-            String oldPathType = remoteCustomData.put(RemoteStorePathType.NAME, newPathType);
-            logger.trace(() -> new ParameterizedMessage("Added new path type {}, replaced old path type {}", newPathType, oldPathType));
-            tmpImdBuilder.putCustom(IndexMetadata.REMOTE_STORE_CUSTOM_KEY, remoteCustomData);
+    /**
+     * Adds the remote store path type information in custom data of index metadata.
+     *
+     * @param tmpImdBuilder     index metadata builder.
+     * @param assertNullOldType flag to verify that the old remote store path type is null
+     */
+    public void addRemoteStorePathStrategyInCustomData(IndexMetadata.Builder tmpImdBuilder, boolean assertNullOldType) {
+        if (remoteStorePathStrategyResolver == null) {
+            return;
+        }
+        // It is possible that remote custom data exists already. In such cases, we need to only update the path type
+        // in the remote store custom data map.
+        Map<String, String> existingCustomData = tmpImdBuilder.removeCustom(IndexMetadata.REMOTE_STORE_CUSTOM_KEY);
+        assert assertNullOldType == false || Objects.isNull(existingCustomData);
+
+        // Determine the path type for use using the remoteStorePathResolver.
+        RemoteStorePathStrategy newPathStrategy = remoteStorePathStrategyResolver.get();
+        Map<String, String> remoteCustomData = new HashMap<>();
+        remoteCustomData.put(PathType.NAME, newPathStrategy.getType().name());
+        if (Objects.nonNull(newPathStrategy.getHashAlgorithm())) {
+            remoteCustomData.put(PathHashAlgorithm.NAME, newPathStrategy.getHashAlgorithm().name());
         }
+        logger.trace(() -> new ParameterizedMessage("Added newStrategy={}, replaced oldStrategy={}", remoteCustomData, existingCustomData));
+        tmpImdBuilder.putCustom(IndexMetadata.REMOTE_STORE_CUSTOM_KEY, remoteCustomData);
     }
 
     private ClusterState applyCreateIndexRequestWithV1Templates(
@@ -930,8 +949,8 @@ static Settings aggregateIndexSettings(
         indexSettingsBuilder.put(IndexMetadata.SETTING_INDEX_PROVIDED_NAME, request.getProvidedName());
         indexSettingsBuilder.put(SETTING_INDEX_UUID, UUIDs.randomBase64UUID());
 
-        updateReplicationStrategy(indexSettingsBuilder, request.settings(), settings, combinedTemplateSettings);
-        updateRemoteStoreSettings(indexSettingsBuilder, settings);
+        updateReplicationStrategy(indexSettingsBuilder, request.settings(), settings, combinedTemplateSettings, clusterSettings);
+        updateRemoteStoreSettings(indexSettingsBuilder, currentState, clusterSettings, settings, request.index());
 
         if (sourceMetadata != null) {
             assert request.resizeType() != null;
@@ -979,29 +998,34 @@ static Settings aggregateIndexSettings(
      * @param clusterSettings cluster level settings
      * @param combinedTemplateSettings combined template settings which satisfy the index
      */
-    private static void updateReplicationStrategy(
+    public static void updateReplicationStrategy(
         Settings.Builder settingsBuilder,
         Settings requestSettings,
-        Settings clusterSettings,
-        Settings combinedTemplateSettings
+        Settings nodeSettings,
+        Settings combinedTemplateSettings,
+        ClusterSettings clusterSettings
     ) {
         // The replication setting is applied in the following order:
-        // 1. Explicit index creation request parameter
-        // 2. Template property for replication type
-        // 3. Defaults to segment if remote store attributes on the cluster
-        // 4. Default cluster level setting
+        // 1. Strictly SEGMENT if cluster is undergoing remote store migration
+        // 2. Explicit index creation request parameter
+        // 3. Template property for replication type
+        // 4. Replication type according to cluster level settings
+        // 5. Defaults to segment if remote store attributes on the cluster
+        // 6. Default cluster level setting
 
         final ReplicationType indexReplicationType;
-        if (INDEX_REPLICATION_TYPE_SETTING.exists(requestSettings)) {
+        if (isMigratingToRemoteStore(clusterSettings)) {
+            indexReplicationType = ReplicationType.SEGMENT;
+        } else if (INDEX_REPLICATION_TYPE_SETTING.exists(requestSettings)) {
             indexReplicationType = INDEX_REPLICATION_TYPE_SETTING.get(requestSettings);
-        } else if (INDEX_REPLICATION_TYPE_SETTING.exists(combinedTemplateSettings)) {
+        } else if (combinedTemplateSettings != null && INDEX_REPLICATION_TYPE_SETTING.exists(combinedTemplateSettings)) {
             indexReplicationType = INDEX_REPLICATION_TYPE_SETTING.get(combinedTemplateSettings);
-        } else if (CLUSTER_REPLICATION_TYPE_SETTING.exists(clusterSettings)) {
-            indexReplicationType = CLUSTER_REPLICATION_TYPE_SETTING.get(clusterSettings);
-        } else if (isRemoteDataAttributePresent(clusterSettings)) {
+        } else if (CLUSTER_REPLICATION_TYPE_SETTING.exists(nodeSettings)) {
+            indexReplicationType = CLUSTER_REPLICATION_TYPE_SETTING.get(nodeSettings);
+        } else if (isRemoteDataAttributePresent(nodeSettings)) {
             indexReplicationType = ReplicationType.SEGMENT;
         } else {
-            indexReplicationType = CLUSTER_REPLICATION_TYPE_SETTING.getDefault(clusterSettings);
+            indexReplicationType = CLUSTER_REPLICATION_TYPE_SETTING.getDefault(nodeSettings);
         }
         settingsBuilder.put(SETTING_REPLICATION_TYPE, indexReplicationType);
     }
@@ -1009,23 +1033,49 @@ private static void updateReplicationStrategy(
     /**
      * Updates index settings to enable remote store by default based on node attributes
      * @param settingsBuilder index settings builder to be updated with relevant settings
+     * @param clusterState state of cluster
      * @param clusterSettings cluster level settings
+     * @param nodeSettings node level settings
+     * @param indexName name of index
      */
-    private static void updateRemoteStoreSettings(Settings.Builder settingsBuilder, Settings clusterSettings) {
-        if (isRemoteDataAttributePresent(clusterSettings)) {
-            settingsBuilder.put(SETTING_REMOTE_STORE_ENABLED, true)
-                .put(
-                    SETTING_REMOTE_SEGMENT_STORE_REPOSITORY,
-                    clusterSettings.get(
-                        Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY
-                    )
-                )
-                .put(
-                    SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY,
-                    clusterSettings.get(
-                        Node.NODE_ATTRIBUTES.getKey() + RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY
-                    )
-                );
+    public static void updateRemoteStoreSettings(
+        Settings.Builder settingsBuilder,
+        ClusterState clusterState,
+        ClusterSettings clusterSettings,
+        Settings nodeSettings,
+        String indexName
+    ) {
+        if ((isRemoteDataAttributePresent(nodeSettings)
+            && clusterSettings.get(REMOTE_STORE_COMPATIBILITY_MODE_SETTING).equals(RemoteStoreNodeService.CompatibilityMode.STRICT))
+            || isMigratingToRemoteStore(clusterSettings)) {
+            String segmentRepo, translogRepo;
+
+            Optional<DiscoveryNode> remoteNode = clusterState.nodes()
+                .getNodes()
+                .values()
+                .stream()
+                .filter(DiscoveryNode::isRemoteStoreNode)
+                .findFirst();
+
+            if (remoteNode.isPresent()) {
+                translogRepo = remoteNode.get()
+                    .getAttributes()
+                    .get(RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY);
+                segmentRepo = remoteNode.get()
+                    .getAttributes()
+                    .get(RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY);
+                if (segmentRepo != null && translogRepo != null) {
+                    settingsBuilder.put(SETTING_REMOTE_STORE_ENABLED, true)
+                        .put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, segmentRepo)
+                        .put(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, translogRepo);
+                } else {
+                    ValidationException validationException = new ValidationException();
+                    validationException.addValidationErrors(
+                        Collections.singletonList("Cluster is migrating to remote store but no remote node found, failing index creation")
+                    );
+                    throw new IndexCreationException(indexName, validationException);
+                }
+            }
         }
     }
 
@@ -1586,10 +1636,15 @@ public static void validateTranslogRetentionSettings(Settings indexSettings) {
      * @param clusterSettings cluster setting
      */
     public static void validateRefreshIntervalSettings(Settings requestSettings, ClusterSettings clusterSettings) {
-        if (IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.exists(requestSettings) == false) {
+        if (IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.exists(requestSettings) == false
+            || requestSettings.get(IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.getKey()) == null) {
             return;
         }
         TimeValue requestRefreshInterval = IndexSettings.INDEX_REFRESH_INTERVAL_SETTING.get(requestSettings);
+        // If the refresh interval supplied is -1, we allow the index to be created because -1 means no periodic refresh.
+        if (requestRefreshInterval.millis() == -1) {
+            return;
+        }
         TimeValue clusterMinimumRefreshInterval = clusterSettings.get(IndicesService.CLUSTER_MINIMUM_INDEX_REFRESH_INTERVAL_SETTING);
         if (requestRefreshInterval.millis() < clusterMinimumRefreshInterval.millis()) {
             throw new IllegalArgumentException(
@@ -1609,7 +1664,7 @@ public static void validateRefreshIntervalSettings(Settings requestSettings, Clu
      * @param clusterSettings cluster setting
      */
     static void validateTranslogDurabilitySettings(Settings requestSettings, ClusterSettings clusterSettings, Settings settings) {
-        if (isRemoteDataAttributePresent(settings) == false
+        if ((isRemoteDataAttributePresent(settings) == false && isMigratingToRemoteStore(clusterSettings) == false)
             || IndexSettings.INDEX_TRANSLOG_DURABILITY_SETTING.exists(requestSettings) == false
             || clusterSettings.get(IndicesService.CLUSTER_REMOTE_INDEX_RESTRICT_ASYNC_DURABILITY_SETTING) == false) {
             return;
diff --git a/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java b/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java
index faadc3f7583fb..7c179f6d4d8fd 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/IndexRoutingTable.java
@@ -302,6 +302,20 @@ public List<ShardRouting> shardsWithState(ShardRoutingState state) {
         return shards;
     }
 
+    /**
+     * Returns a {@link List} of shards that match the provided {@link Predicate}
+     *
+     * @param predicate {@link Predicate} to apply
+     * @return a {@link List} of shards that match one of the given {@link Predicate}
+     */
+    public List<ShardRouting> shardsMatchingPredicate(Predicate<ShardRouting> predicate) {
+        List<ShardRouting> shards = new ArrayList<>();
+        for (IndexShardRoutingTable shardRoutingTable : this) {
+            shards.addAll(shardRoutingTable.shardsMatchingPredicate(predicate));
+        }
+        return shards;
+    }
+
     public int shardsMatchingPredicateCount(Predicate<ShardRouting> predicate) {
         int count = 0;
         for (IndexShardRoutingTable shardRoutingTable : this) {
diff --git a/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java b/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java
index 36149d014ea84..fd8cbea42c12f 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/IndexShardRoutingTable.java
@@ -34,7 +34,6 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.opensearch.cluster.metadata.WeightedRoutingMetadata;
 import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.cluster.node.DiscoveryNodes;
 import org.opensearch.common.Nullable;
@@ -63,7 +62,6 @@
 import java.util.Set;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 import static java.util.Collections.emptyMap;
 
@@ -96,8 +94,8 @@ public class IndexShardRoutingTable implements Iterable<ShardRouting> {
     private volatile Map<AttributesKey, AttributesRoutings> initializingShardsByAttributes = emptyMap();
     private final Object shardsByAttributeMutex = new Object();
     private final Object shardsByWeightMutex = new Object();
-    private volatile Map<WeightedRoutingKey, List<ShardRouting>> activeShardsByWeight = emptyMap();
-    private volatile Map<WeightedRoutingKey, List<ShardRouting>> initializingShardsByWeight = emptyMap();
+    private volatile Map<WeightedRoutingKey, WeightedShardRoutings> activeShardsByWeight = emptyMap();
+    private volatile Map<WeightedRoutingKey, WeightedShardRoutings> initializingShardsByWeight = emptyMap();
 
     private static final Logger logger = LogManager.getLogger(IndexShardRoutingTable.class);
 
@@ -249,7 +247,7 @@ public List<ShardRouting> assignedShards() {
         return this.assignedShards;
     }
 
-    public Map<WeightedRoutingKey, List<ShardRouting>> getActiveShardsByWeight() {
+    public Map<WeightedRoutingKey, WeightedShardRoutings> getActiveShardsByWeight() {
         return activeShardsByWeight;
     }
 
@@ -338,23 +336,7 @@ public ShardIterator activeInitializingShardsWeightedIt(
         // append shards for attribute value with weight zero, so that shard search requests can be tried on
         // shard copies in case of request failure from other attribute values.
         if (isFailOpenEnabled) {
-            try {
-                Stream<String> keys = weightedRouting.weights()
-                    .entrySet()
-                    .stream()
-                    .filter(entry -> entry.getValue().intValue() == WeightedRoutingMetadata.WEIGHED_AWAY_WEIGHT)
-                    .map(Map.Entry::getKey);
-                keys.forEach(key -> {
-                    ShardIterator iterator = onlyNodeSelectorActiveInitializingShardsIt(weightedRouting.attributeName() + ":" + key, nodes);
-                    while (iterator.remaining() > 0) {
-                        ordered.add(iterator.nextOrNull());
-                    }
-                });
-            } catch (IllegalArgumentException e) {
-                // this exception is thrown by {@link onlyNodeSelectorActiveInitializingShardsIt} in case count of shard
-                // copies found is zero
-                logger.debug("no shard copies found for shard id [{}] for node attribute with weight zero", shardId);
-            }
+            ordered.addAll(activeInitializingShardsWithoutWeights(weightedRouting, nodes, defaultWeight));
         }
 
         return new PlainShardIterator(shardId, ordered);
@@ -378,6 +360,18 @@ private List<ShardRouting> activeInitializingShardsWithWeights(
         return orderedListWithDistinctShards;
     }
 
+    private List<ShardRouting> activeInitializingShardsWithoutWeights(
+        WeightedRouting weightedRouting,
+        DiscoveryNodes nodes,
+        double defaultWeight
+    ) {
+        List<ShardRouting> ordered = new ArrayList<>(getActiveShardsWithoutWeight(weightedRouting, nodes, defaultWeight));
+        if (!allInitializingShards.isEmpty()) {
+            ordered.addAll(getInitializingShardsWithoutWeight(weightedRouting, nodes, defaultWeight));
+        }
+        return ordered.stream().distinct().collect(Collectors.toList());
+    }
+
     /**
      * Returns a list containing shard routings ordered using weighted round-robin scheduling.
      */
@@ -910,6 +904,22 @@ public List<ShardRouting> shardsWithState(ShardRoutingState state) {
         return shards;
     }
 
+    /**
+     * Returns a {@link List} of shards that match the provided {@link Predicate}
+     *
+     * @param predicate {@link Predicate} to apply
+     * @return a {@link List} of shards that match one of the given {@link Predicate}
+     */
+    public List<ShardRouting> shardsMatchingPredicate(Predicate<ShardRouting> predicate) {
+        List<ShardRouting> shards = new ArrayList<>();
+        for (ShardRouting shardEntry : this) {
+            if (predicate.test(shardEntry)) {
+                shards.add(shardEntry);
+            }
+        }
+        return shards;
+    }
+
     public int shardsMatchingPredicateCount(Predicate<ShardRouting> predicate) {
         int count = 0;
         for (ShardRouting shardEntry : this) {
@@ -949,20 +959,60 @@ public int hashCode() {
         }
     }
 
+    /**
+     * Holder class for shard routing(s) which are classified and stored based on their weights.
+     *
+     * @opensearch.api
+     */
+    @PublicApi(since = "2.14.0")
+    public static class WeightedShardRoutings {
+        private final List<ShardRouting> shardRoutingsWithWeight;
+        private final List<ShardRouting> shardRoutingWithoutWeight;
+
+        public WeightedShardRoutings(List<ShardRouting> shardRoutingsWithWeight, List<ShardRouting> shardRoutingWithoutWeight) {
+            this.shardRoutingsWithWeight = Collections.unmodifiableList(shardRoutingsWithWeight);
+            this.shardRoutingWithoutWeight = Collections.unmodifiableList(shardRoutingWithoutWeight);
+        }
+
+        public List<ShardRouting> getShardRoutingsWithWeight() {
+            return shardRoutingsWithWeight;
+        }
+
+        public List<ShardRouting> getShardRoutingWithoutWeight() {
+            return shardRoutingWithoutWeight;
+        }
+    }
+
     /**
      * *
      * Gets active shard routing from memory if available, else calculates and put it in memory.
      */
     private List<ShardRouting> getActiveShardsByWeight(WeightedRouting weightedRouting, DiscoveryNodes nodes, double defaultWeight) {
         WeightedRoutingKey key = new WeightedRoutingKey(weightedRouting);
-        List<ShardRouting> shardRoutings = activeShardsByWeight.get(key);
-        if (shardRoutings == null) {
-            synchronized (shardsByWeightMutex) {
-                shardRoutings = shardsOrderedByWeight(activeShards, weightedRouting, nodes, defaultWeight);
-                activeShardsByWeight = new MapBuilder().put(key, shardRoutings).immutableMap();
-            }
+        if (activeShardsByWeight.get(key) == null) {
+            populateActiveShardWeightsMap(weightedRouting, nodes, defaultWeight);
+        }
+        return activeShardsByWeight.get(key).getShardRoutingsWithWeight();
+    }
+
+    private List<ShardRouting> getActiveShardsWithoutWeight(WeightedRouting weightedRouting, DiscoveryNodes nodes, double defaultWeight) {
+        WeightedRoutingKey key = new WeightedRoutingKey(weightedRouting);
+        if (activeShardsByWeight.get(key) == null) {
+            populateActiveShardWeightsMap(weightedRouting, nodes, defaultWeight);
+        }
+        return activeShardsByWeight.get(key).getShardRoutingWithoutWeight();
+    }
+
+    private void populateActiveShardWeightsMap(WeightedRouting weightedRouting, DiscoveryNodes nodes, double defaultWeight) {
+        WeightedRoutingKey key = new WeightedRoutingKey(weightedRouting);
+        List<ShardRouting> weightedRoutings = shardsOrderedByWeight(activeShards, weightedRouting, nodes, defaultWeight);
+        List<ShardRouting> nonWeightedRoutings = activeShards.stream()
+            .filter(shard -> !weightedRoutings.contains(shard))
+            .collect(Collectors.toUnmodifiableList());
+        synchronized (shardsByWeightMutex) {
+            activeShardsByWeight = new MapBuilder().put(key, new WeightedShardRoutings(weightedRoutings, nonWeightedRoutings))
+                .immutableMap();
         }
-        return shardRoutings;
     }
 
     /**
@@ -971,14 +1021,34 @@ private List<ShardRouting> getActiveShardsByWeight(WeightedRouting weightedRouti
      */
     private List<ShardRouting> getInitializingShardsByWeight(WeightedRouting weightedRouting, DiscoveryNodes nodes, double defaultWeight) {
         WeightedRoutingKey key = new WeightedRoutingKey(weightedRouting);
-        List<ShardRouting> shardRoutings = initializingShardsByWeight.get(key);
-        if (shardRoutings == null) {
-            synchronized (shardsByWeightMutex) {
-                shardRoutings = shardsOrderedByWeight(activeShards, weightedRouting, nodes, defaultWeight);
-                initializingShardsByWeight = new MapBuilder().put(key, shardRoutings).immutableMap();
-            }
+        if (initializingShardsByWeight.get(key) == null) {
+            populateInitializingShardWeightsMap(weightedRouting, nodes, defaultWeight);
+        }
+        return initializingShardsByWeight.get(key).getShardRoutingsWithWeight();
+    }
+
+    private List<ShardRouting> getInitializingShardsWithoutWeight(
+        WeightedRouting weightedRouting,
+        DiscoveryNodes nodes,
+        double defaultWeight
+    ) {
+        WeightedRoutingKey key = new WeightedRoutingKey(weightedRouting);
+        if (initializingShardsByWeight.get(key) == null) {
+            populateInitializingShardWeightsMap(weightedRouting, nodes, defaultWeight);
+        }
+        return initializingShardsByWeight.get(key).getShardRoutingWithoutWeight();
+    }
+
+    private void populateInitializingShardWeightsMap(WeightedRouting weightedRouting, DiscoveryNodes nodes, double defaultWeight) {
+        WeightedRoutingKey key = new WeightedRoutingKey(weightedRouting);
+        List<ShardRouting> weightedRoutings = shardsOrderedByWeight(allInitializingShards, weightedRouting, nodes, defaultWeight);
+        List<ShardRouting> nonWeightedRoutings = allInitializingShards.stream()
+            .filter(shard -> !weightedRoutings.contains(shard))
+            .collect(Collectors.toUnmodifiableList());
+        synchronized (shardsByWeightMutex) {
+            initializingShardsByWeight = new MapBuilder().put(key, new WeightedShardRoutings(weightedRoutings, nonWeightedRoutings))
+                .immutableMap();
         }
-        return shardRoutings;
     }
 
     /**
diff --git a/server/src/main/java/org/opensearch/cluster/routing/RoutingNode.java b/server/src/main/java/org/opensearch/cluster/routing/RoutingNode.java
index 15ec41d5c3fbb..24c3077960444 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/RoutingNode.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/RoutingNode.java
@@ -204,6 +204,10 @@ public int size() {
         return shards.size();
     }
 
+    public Collection<ShardRouting> getInitializingShards() {
+        return initializingShards;
+    }
+
     /**
      * Add a new shard to this node
      * @param shard Shard to create on this Node
diff --git a/server/src/main/java/org/opensearch/cluster/routing/RoutingNodes.java b/server/src/main/java/org/opensearch/cluster/routing/RoutingNodes.java
index 938a603c459c9..ab455f52c4195 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/RoutingNodes.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/RoutingNodes.java
@@ -67,6 +67,8 @@
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+import static org.opensearch.node.remotestore.RemoteStoreNodeService.isMigratingToRemoteStore;
+
 /**
  * {@link RoutingNodes} represents a copy the routing information contained in the {@link ClusterState cluster state}.
  * It can be either initialized as mutable or immutable (see {@link #RoutingNodes(ClusterState, boolean)}), allowing
@@ -418,6 +420,20 @@ public ShardRouting activeReplicaWithOldestVersion(ShardId shardId) {
             .orElse(null);
     }
 
+    /**
+     * Returns one active replica shard on a remote node for the given shard id or <code>null</code> if
+     * no such replica is found.
+     * <p>
+     * Since we aim to continue moving forward during remote store migration, replicas already migrated to remote nodes
+     * are preferred for primary promotion
+     */
+    public ShardRouting activeReplicaOnRemoteNode(ShardId shardId) {
+        return assignedShards(shardId).stream().filter(shr -> !shr.primary() && shr.active()).filter((shr) -> {
+            RoutingNode nd = node(shr.currentNodeId());
+            return (nd != null && nd.node().isRemoteStoreNode());
+        }).findFirst().orElse(null);
+    }
+
     /**
      * Returns <code>true</code> iff all replicas are active for the given shard routing. Otherwise <code>false</code>
      */
@@ -735,11 +751,17 @@ private void unassignPrimaryAndPromoteActiveReplicaIfExists(
         RoutingChangesObserver routingChangesObserver
     ) {
         assert failedShard.primary();
-        ShardRouting activeReplica;
-        if (metadata.isSegmentReplicationEnabled(failedShard.getIndexName())) {
-            activeReplica = activeReplicaWithOldestVersion(failedShard.shardId());
-        } else {
-            activeReplica = activeReplicaWithHighestVersion(failedShard.shardId());
+        ShardRouting activeReplica = null;
+        if (isMigratingToRemoteStore(metadata)) {
+            // we might not find any replica on remote node
+            activeReplica = activeReplicaOnRemoteNode(failedShard.shardId());
+        }
+        if (activeReplica == null) {
+            if (metadata.isSegmentReplicationEnabled(failedShard.getIndexName())) {
+                activeReplica = activeReplicaWithOldestVersion(failedShard.shardId());
+            } else {
+                activeReplica = activeReplicaWithHighestVersion(failedShard.shardId());
+            }
         }
         if (activeReplica == null) {
             moveToUnassigned(failedShard, unassignedInfo);
diff --git a/server/src/main/java/org/opensearch/cluster/routing/WeightedRouting.java b/server/src/main/java/org/opensearch/cluster/routing/WeightedRouting.java
index 468fac08d2946..6f0e4fe90cfff 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/WeightedRouting.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/WeightedRouting.java
@@ -14,6 +14,7 @@
 import org.opensearch.core.common.io.stream.Writeable;
 
 import java.io.IOException;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
@@ -25,27 +26,26 @@
  */
 @PublicApi(since = "2.4.0")
 public class WeightedRouting implements Writeable {
-    private String attributeName;
-    private Map<String, Double> weights;
+    private final String attributeName;
+    private final Map<String, Double> weights;
+    private final int hashCode;
 
     public WeightedRouting() {
-        this.attributeName = "";
-        this.weights = new HashMap<>(3);
+        this("", new HashMap<>(3));
     }
 
     public WeightedRouting(String attributeName, Map<String, Double> weights) {
         this.attributeName = attributeName;
-        this.weights = weights;
+        this.weights = Collections.unmodifiableMap(weights);
+        this.hashCode = Objects.hash(this.attributeName, this.weights);
     }
 
     public WeightedRouting(WeightedRouting weightedRouting) {
-        this.attributeName = weightedRouting.attributeName();
-        this.weights = weightedRouting.weights;
+        this(weightedRouting.attributeName(), weightedRouting.weights);
     }
 
     public WeightedRouting(StreamInput in) throws IOException {
-        attributeName = in.readString();
-        weights = (Map<String, Double>) in.readGenericValue();
+        this(in.readString(), (Map<String, Double>) in.readGenericValue());
     }
 
     public boolean isSet() {
@@ -70,7 +70,7 @@ public boolean equals(Object o) {
 
     @Override
     public int hashCode() {
-        return Objects.hash(attributeName, weights);
+        return hashCode;
     }
 
     @Override
diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/AllocationConstraints.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/AllocationConstraints.java
index 5375910c57579..6702db4b43e91 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/allocation/AllocationConstraints.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/AllocationConstraints.java
@@ -30,9 +30,9 @@ public class AllocationConstraints {
 
     public AllocationConstraints() {
         this.constraints = new HashMap<>();
-        this.constraints.putIfAbsent(INDEX_SHARD_PER_NODE_BREACH_CONSTRAINT_ID, new Constraint(isIndexShardsPerNodeBreached()));
-        this.constraints.putIfAbsent(INDEX_PRIMARY_SHARD_BALANCE_CONSTRAINT_ID, new Constraint(isPerIndexPrimaryShardsPerNodeBreached()));
-        this.constraints.putIfAbsent(CLUSTER_PRIMARY_SHARD_BALANCE_CONSTRAINT_ID, new Constraint(isPrimaryShardsPerNodeBreached()));
+        this.constraints.put(INDEX_SHARD_PER_NODE_BREACH_CONSTRAINT_ID, new Constraint(isIndexShardsPerNodeBreached()));
+        this.constraints.put(INDEX_PRIMARY_SHARD_BALANCE_CONSTRAINT_ID, new Constraint(isPerIndexPrimaryShardsPerNodeBreached()));
+        this.constraints.put(CLUSTER_PRIMARY_SHARD_BALANCE_CONSTRAINT_ID, new Constraint(isPrimaryShardsPerNodeBreached(0.0f)));
     }
 
     public void updateAllocationConstraint(String constraint, boolean enable) {
diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/AllocationService.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/AllocationService.java
index fde6854b6155c..0c8207a4ac49e 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/allocation/AllocationService.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/AllocationService.java
@@ -35,7 +35,9 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.opensearch.Version;
 import org.opensearch.cluster.ClusterInfoService;
+import org.opensearch.cluster.ClusterManagerMetrics;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.RestoreInProgress;
 import org.opensearch.cluster.health.ClusterHealthStatus;
@@ -54,12 +56,12 @@
 import org.opensearch.cluster.routing.allocation.command.AllocationCommands;
 import org.opensearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.opensearch.cluster.routing.allocation.decider.Decision;
+import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.gateway.GatewayAllocator;
 import org.opensearch.gateway.PriorityComparator;
+import org.opensearch.gateway.ShardsBatchGatewayAllocator;
 import org.opensearch.snapshots.SnapshotsInfoService;
-import org.opensearch.telemetry.metrics.Histogram;
-import org.opensearch.telemetry.metrics.MetricsRegistry;
 import org.opensearch.telemetry.metrics.noop.NoopMetricsRegistry;
 
 import java.util.ArrayList;
@@ -77,6 +79,7 @@
 import static java.util.Collections.emptyList;
 import static java.util.Collections.singletonList;
 import static org.opensearch.cluster.routing.UnassignedInfo.INDEX_DELAYED_NODE_LEFT_TIMEOUT_SETTING;
+import static org.opensearch.cluster.routing.allocation.ExistingShardsAllocator.EXISTING_SHARDS_ALLOCATOR_BATCH_MODE;
 
 /**
  * This service manages the node allocation of a cluster. For this reason the
@@ -91,11 +94,12 @@ public class AllocationService {
     private static final Logger logger = LogManager.getLogger(AllocationService.class);
 
     private final AllocationDeciders allocationDeciders;
+    private Settings settings;
     private Map<String, ExistingShardsAllocator> existingShardsAllocators;
     private final ShardsAllocator shardsAllocator;
     private final ClusterInfoService clusterInfoService;
     private SnapshotsInfoService snapshotsInfoService;
-    private Histogram rerouteHistogram;
+    private final ClusterManagerMetrics clusterManagerMetrics;
 
     // only for tests that use the GatewayAllocator as the unique ExistingShardsAllocator
     public AllocationService(
@@ -105,7 +109,13 @@ public AllocationService(
         ClusterInfoService clusterInfoService,
         SnapshotsInfoService snapshotsInfoService
     ) {
-        this(allocationDeciders, shardsAllocator, clusterInfoService, snapshotsInfoService, NoopMetricsRegistry.INSTANCE);
+        this(
+            allocationDeciders,
+            shardsAllocator,
+            clusterInfoService,
+            snapshotsInfoService,
+            new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE)
+        );
         setExistingShardsAllocators(Collections.singletonMap(GatewayAllocator.ALLOCATOR_NAME, gatewayAllocator));
     }
 
@@ -114,21 +124,25 @@ public AllocationService(
         ShardsAllocator shardsAllocator,
         ClusterInfoService clusterInfoService,
         SnapshotsInfoService snapshotsInfoService,
-        MetricsRegistry metricsRegistry
+        ClusterManagerMetrics clusterManagerMetrics
+    ) {
+        this(allocationDeciders, shardsAllocator, clusterInfoService, snapshotsInfoService, Settings.EMPTY, clusterManagerMetrics);
+    }
+
+    public AllocationService(
+        AllocationDeciders allocationDeciders,
+        ShardsAllocator shardsAllocator,
+        ClusterInfoService clusterInfoService,
+        SnapshotsInfoService snapshotsInfoService,
+        Settings settings,
+        ClusterManagerMetrics clusterManagerMetrics
     ) {
         this.allocationDeciders = allocationDeciders;
         this.shardsAllocator = shardsAllocator;
         this.clusterInfoService = clusterInfoService;
         this.snapshotsInfoService = snapshotsInfoService;
-        initializeMetrics(metricsRegistry);
-    }
-
-    private void initializeMetrics(MetricsRegistry metricsRegistry) {
-        this.rerouteHistogram = metricsRegistry.createHistogram(
-            "allocation.reroute.latency",
-            "Histogram for recording latency of shard re-routing",
-            "ms"
-        );
+        this.settings = settings;
+        this.clusterManagerMetrics = clusterManagerMetrics;
     }
 
     /**
@@ -553,7 +567,10 @@ private void reroute(RoutingAllocation allocation) {
 
         allocateExistingUnassignedShards(allocation);  // try to allocate existing shard copies first
         shardsAllocator.allocate(allocation);
-        this.rerouteHistogram.record((double) Math.max(0, TimeValue.nsecToMSec(System.nanoTime() - rerouteStartTimeNS)));
+        ClusterManagerMetrics.recordLatency(
+            clusterManagerMetrics.rerouteHistogram,
+            (double) Math.max(0, TimeValue.nsecToMSec(System.nanoTime() - rerouteStartTimeNS))
+        );
         assert RoutingNodes.assertShardStats(allocation.routingNodes());
     }
 
@@ -564,6 +581,22 @@ private void allocateExistingUnassignedShards(RoutingAllocation allocation) {
             existingShardsAllocator.beforeAllocation(allocation);
         }
 
+        /*
+         Use batch mode if enabled and there is no custom allocator set for Allocation service
+         */
+        Boolean batchModeEnabled = EXISTING_SHARDS_ALLOCATOR_BATCH_MODE.get(settings);
+        if (batchModeEnabled
+            && allocation.nodes().getMinNodeVersion().onOrAfter(Version.V_2_14_0)
+            && existingShardsAllocators.size() == 2) {
+            /*
+             If we do not have any custom allocator set then we will be using ShardsBatchGatewayAllocator
+             Currently AllocationService will not run any custom Allocator that implements allocateAllUnassignedShards
+             */
+            allocateAllUnassignedShards(allocation);
+            return;
+        }
+        logger.warn("Falling back to single shard assignment since batch mode disable or multiple custom allocators set");
+
         final RoutingNodes.UnassignedShards.UnassignedIterator primaryIterator = allocation.routingNodes().unassigned().iterator();
         while (primaryIterator.hasNext()) {
             final ShardRouting shardRouting = primaryIterator.next();
@@ -585,6 +618,14 @@ private void allocateExistingUnassignedShards(RoutingAllocation allocation) {
         }
     }
 
+    private void allocateAllUnassignedShards(RoutingAllocation allocation) {
+        ExistingShardsAllocator allocator = existingShardsAllocators.get(ShardsBatchGatewayAllocator.ALLOCATOR_NAME);
+        allocator.allocateAllUnassignedShards(allocation, true);
+        allocator.afterPrimariesBeforeReplicas(allocation);
+        // Replicas Assignment
+        allocator.allocateAllUnassignedShards(allocation, false);
+    }
+
     private void disassociateDeadNodes(RoutingAllocation allocation) {
         for (Iterator<RoutingNode> it = allocation.routingNodes().mutableIterator(); it.hasNext();) {
             RoutingNode node = it.next();
diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/ConstraintTypes.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/ConstraintTypes.java
index ae2d4a0926194..08fe8f92d1f80 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/allocation/ConstraintTypes.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/ConstraintTypes.java
@@ -28,6 +28,11 @@ public class ConstraintTypes {
      */
     public final static String CLUSTER_PRIMARY_SHARD_BALANCE_CONSTRAINT_ID = "cluster.primary.shard.balance.constraint";
 
+    /**
+     * Defines a cluster constraint which is breached when a node contains more than avg primary shards across all indices
+     */
+    public final static String CLUSTER_PRIMARY_SHARD_REBALANCE_CONSTRAINT_ID = "cluster.primary.shard.rebalance.constraint";
+
     /**
      * Defines an index constraint which is breached when a node contains more than avg number of shards for an index
      */
@@ -70,14 +75,14 @@ public static Predicate<Constraint.ConstraintParams> isPerIndexPrimaryShardsPerN
     }
 
     /**
-     * Defines a predicate which returns true when a node contains more than average number of primary shards. This
-     * constraint is used in weight calculation during allocation only. When breached a high weight {@link ConstraintTypes#CONSTRAINT_WEIGHT}
-     * is assigned to node resulting in lesser chances of node being selected as allocation target
+     * Defines a predicate which returns true when a node contains more than average number of primary shards with added buffer. This
+     * constraint is used in weight calculation during allocation/rebalance both. When breached a high weight {@link ConstraintTypes#CONSTRAINT_WEIGHT}
+     * is assigned to node resulting in lesser chances of node being selected as allocation/rebalance target
      */
-    public static Predicate<Constraint.ConstraintParams> isPrimaryShardsPerNodeBreached() {
+    public static Predicate<Constraint.ConstraintParams> isPrimaryShardsPerNodeBreached(float buffer) {
         return (params) -> {
             int primaryShardCount = params.getNode().numPrimaryShards();
-            int allowedPrimaryShardCount = (int) Math.ceil(params.getBalancer().avgPrimaryShardsPerNode());
+            int allowedPrimaryShardCount = (int) Math.ceil(params.getBalancer().avgPrimaryShardsPerNode() * (1 + buffer));
             return primaryShardCount >= allowedPrimaryShardCount;
         };
     }
diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/ExistingShardsAllocator.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/ExistingShardsAllocator.java
index f1889cdf780d4..fb2a37237f8b6 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/allocation/ExistingShardsAllocator.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/ExistingShardsAllocator.java
@@ -39,12 +39,13 @@
 import org.opensearch.common.Nullable;
 import org.opensearch.common.settings.Setting;
 import org.opensearch.gateway.GatewayAllocator;
+import org.opensearch.gateway.ShardsBatchGatewayAllocator;
 
 import java.util.List;
 
 /**
  * Searches for, and allocates, shards for which there is an existing on-disk copy somewhere in the cluster. The default implementation is
- * {@link GatewayAllocator}, but plugins can supply their own implementations too.
+ * {@link GatewayAllocator} and {@link ShardsBatchGatewayAllocator}, but plugins can supply their own implementations too.
  *
  * @opensearch.internal
  */
@@ -60,6 +61,26 @@ public interface ExistingShardsAllocator {
         Setting.Property.PrivateIndex
     );
 
+    /**
+     * Boolean setting to enable/disable batch allocation of unassigned shards already existing on disk.
+     * This will allow sending all Unassigned Shards to the ExistingShard Allocator to  make decision to allocate
+     * in one or more go.
+     *
+     * Enable this setting if your ExistingShardAllocator is implementing the
+     * {@link ExistingShardsAllocator#allocateAllUnassignedShards(RoutingAllocation, boolean)} method.
+     * The default implementation of this method is not optimized and assigns shards one by one.
+     *
+     * If no plugin overrides {@link ExistingShardsAllocator} then default implementation will be use for it , i.e,
+     * {@link ShardsBatchGatewayAllocator}.
+     *
+     * This setting is experimental at this point.
+     */
+    Setting<Boolean> EXISTING_SHARDS_ALLOCATOR_BATCH_MODE = Setting.boolSetting(
+        "cluster.allocator.existing_shards_allocator.batch_enabled",
+        false,
+        Setting.Property.NodeScope
+    );
+
     /**
      * Called before starting a round of allocation, allowing the allocator to invalidate some caches if appropriate.
      */
@@ -80,6 +101,23 @@ void allocateUnassigned(
         UnassignedAllocationHandler unassignedAllocationHandler
     );
 
+    /**
+     * Allocate all unassigned shards in the given {@link RoutingAllocation} for which this {@link ExistingShardsAllocator} is responsible.
+     * Default implementation calls {@link #allocateUnassigned(ShardRouting, RoutingAllocation, UnassignedAllocationHandler)} for each Unassigned shard
+     * and is kept here for backward compatibility.
+     *
+     * Allocation service will currently run the default implementation of it implemented by {@link ShardsBatchGatewayAllocator}
+     */
+    default void allocateAllUnassignedShards(RoutingAllocation allocation, boolean primary) {
+        RoutingNodes.UnassignedShards.UnassignedIterator iterator = allocation.routingNodes().unassigned().iterator();
+        while (iterator.hasNext()) {
+            ShardRouting shardRouting = iterator.next();
+            if (shardRouting.primary() == primary) {
+                allocateUnassigned(shardRouting, allocation, iterator);
+            }
+        }
+    }
+
     /**
      * Returns an explanation for a single unassigned shard.
      */
diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/IndexMetadataUpdater.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/IndexMetadataUpdater.java
index 7fc78b05880f3..ddcccd597e894 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/allocation/IndexMetadataUpdater.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/IndexMetadataUpdater.java
@@ -32,10 +32,12 @@
 
 package org.opensearch.cluster.routing.allocation;
 
+import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.metadata.Metadata;
+import org.opensearch.cluster.node.DiscoveryNodes;
 import org.opensearch.cluster.routing.IndexShardRoutingTable;
 import org.opensearch.cluster.routing.RecoverySource;
 import org.opensearch.cluster.routing.RoutingChangesObserver;
@@ -45,6 +47,7 @@
 import org.opensearch.common.util.set.Sets;
 import org.opensearch.core.index.Index;
 import org.opensearch.core.index.shard.ShardId;
+import org.opensearch.index.remote.RemoteMigrationIndexMetadataUpdater;
 
 import java.util.Collections;
 import java.util.Comparator;
@@ -67,14 +70,15 @@
  * @opensearch.internal
  */
 public class IndexMetadataUpdater extends RoutingChangesObserver.AbstractRoutingChangesObserver {
+    private final Logger logger = LogManager.getLogger(IndexMetadataUpdater.class);
     private final Map<ShardId, Updates> shardChanges = new HashMap<>();
+    private boolean ongoingRemoteStoreMigration = false;
 
     @Override
     public void shardInitialized(ShardRouting unassignedShard, ShardRouting initializedShard) {
         assert initializedShard.isRelocationTarget() == false : "shardInitialized is not called on relocation target: " + initializedShard;
         if (initializedShard.primary()) {
             increasePrimaryTerm(initializedShard.shardId());
-
             Updates updates = changes(initializedShard.shardId());
             assert updates.initializedPrimary == null : "Primary cannot be initialized more than once in same allocation round: "
                 + "(previous: "
@@ -113,6 +117,12 @@ public void shardFailed(ShardRouting failedShard, UnassignedInfo unassignedInfo)
             }
             increasePrimaryTerm(failedShard.shardId());
         }
+
+        // Track change through shardChanges Map regardless of above-mentioned conditions
+        // To be used to update index metadata while computing new cluster state
+        if (ongoingRemoteStoreMigration) {
+            changes(failedShard.shardId());
+        }
     }
 
     @Override
@@ -120,20 +130,34 @@ public void relocationCompleted(ShardRouting removedRelocationSource) {
         removeAllocationId(removedRelocationSource);
     }
 
+    /**
+     * Adds the target {@link ShardRouting} to the tracking updates set.
+     * Used to track started relocations while applying changes to the new {@link ClusterState}
+     */
+    @Override
+    public void relocationStarted(ShardRouting startedShard, ShardRouting targetRelocatingShard) {
+        // Store change in shardChanges Map regardless of above-mentioned conditions
+        // To be used to update index metadata while computing new cluster state
+        if (ongoingRemoteStoreMigration) {
+            changes(targetRelocatingShard.shardId());
+        }
+    }
+
     /**
      * Updates the current {@link Metadata} based on the changes of this RoutingChangesObserver. Specifically
      * we update {@link IndexMetadata#getInSyncAllocationIds()} and {@link IndexMetadata#primaryTerm(int)} based on
      * the changes made during this allocation.
+     * <br>
+     * Manipulates index settings or index metadata during an ongoing remote store migration
      *
      * @param oldMetadata {@link Metadata} object from before the routing nodes was changed.
      * @param newRoutingTable {@link RoutingTable} object after routing changes were applied.
      * @return adapted {@link Metadata}, potentially the original one if no change was needed.
      */
-    public Metadata applyChanges(Metadata oldMetadata, RoutingTable newRoutingTable) {
+    public Metadata applyChanges(Metadata oldMetadata, RoutingTable newRoutingTable, DiscoveryNodes discoveryNodes) {
         Map<Index, List<Map.Entry<ShardId, Updates>>> changesGroupedByIndex = shardChanges.entrySet()
             .stream()
             .collect(Collectors.groupingBy(e -> e.getKey().getIndex()));
-
         Metadata.Builder metadataBuilder = null;
         for (Map.Entry<Index, List<Map.Entry<ShardId, Updates>>> indexChanges : changesGroupedByIndex.entrySet()) {
             Index index = indexChanges.getKey();
@@ -144,6 +168,17 @@ public Metadata applyChanges(Metadata oldMetadata, RoutingTable newRoutingTable)
                 Updates updates = shardEntry.getValue();
                 indexMetadataBuilder = updateInSyncAllocations(newRoutingTable, oldIndexMetadata, indexMetadataBuilder, shardId, updates);
                 indexMetadataBuilder = updatePrimaryTerm(oldIndexMetadata, indexMetadataBuilder, shardId, updates);
+                if (ongoingRemoteStoreMigration) {
+                    RemoteMigrationIndexMetadataUpdater migrationImdUpdater = new RemoteMigrationIndexMetadataUpdater(
+                        discoveryNodes,
+                        newRoutingTable,
+                        oldIndexMetadata,
+                        oldMetadata.settings(),
+                        logger
+                    );
+                    migrationImdUpdater.maybeUpdateRemoteStorePathStrategy(indexMetadataBuilder, index.getName());
+                    migrationImdUpdater.maybeAddRemoteIndexSettings(indexMetadataBuilder, index.getName());
+                }
             }
 
             if (indexMetadataBuilder != null) {
@@ -369,6 +404,10 @@ private void increasePrimaryTerm(ShardId shardId) {
         changes(shardId).increaseTerm = true;
     }
 
+    public void setOngoingRemoteStoreMigration(boolean ongoingRemoteStoreMigration) {
+        this.ongoingRemoteStoreMigration = ongoingRemoteStoreMigration;
+    }
+
     private static class Updates {
         private boolean increaseTerm; // whether primary term should be increased
         private Set<String> addedAllocationIds = new HashSet<>(); // allocation ids that should be added to the in-sync set
diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/RebalanceConstraints.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/RebalanceConstraints.java
index a4036ec47ec0e..2c2138af18abc 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/allocation/RebalanceConstraints.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/RebalanceConstraints.java
@@ -14,8 +14,10 @@
 import java.util.HashMap;
 import java.util.Map;
 
+import static org.opensearch.cluster.routing.allocation.ConstraintTypes.CLUSTER_PRIMARY_SHARD_REBALANCE_CONSTRAINT_ID;
 import static org.opensearch.cluster.routing.allocation.ConstraintTypes.INDEX_PRIMARY_SHARD_BALANCE_CONSTRAINT_ID;
 import static org.opensearch.cluster.routing.allocation.ConstraintTypes.isPerIndexPrimaryShardsPerNodeBreached;
+import static org.opensearch.cluster.routing.allocation.ConstraintTypes.isPrimaryShardsPerNodeBreached;
 
 /**
  * Constraints applied during rebalancing round; specify conditions which, if breached, reduce the
@@ -27,9 +29,13 @@ public class RebalanceConstraints {
 
     private Map<String, Constraint> constraints;
 
-    public RebalanceConstraints() {
+    public RebalanceConstraints(RebalanceParameter rebalanceParameter) {
         this.constraints = new HashMap<>();
-        this.constraints.putIfAbsent(INDEX_PRIMARY_SHARD_BALANCE_CONSTRAINT_ID, new Constraint(isPerIndexPrimaryShardsPerNodeBreached()));
+        this.constraints.put(INDEX_PRIMARY_SHARD_BALANCE_CONSTRAINT_ID, new Constraint(isPerIndexPrimaryShardsPerNodeBreached()));
+        this.constraints.put(
+            CLUSTER_PRIMARY_SHARD_REBALANCE_CONSTRAINT_ID,
+            new Constraint(isPrimaryShardsPerNodeBreached(rebalanceParameter.getPreferPrimaryBalanceBuffer()))
+        );
     }
 
     public void updateRebalanceConstraint(String constraint, boolean enable) {
diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/RebalanceParameter.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/RebalanceParameter.java
new file mode 100644
index 0000000000000..35fbaede93ba3
--- /dev/null
+++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/RebalanceParameter.java
@@ -0,0 +1,24 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.cluster.routing.allocation;
+
+/**
+ * RebalanceConstraint Params
+ */
+public class RebalanceParameter {
+    private float preferPrimaryBalanceBuffer;
+
+    public RebalanceParameter(float preferPrimaryBalanceBuffer) {
+        this.preferPrimaryBalanceBuffer = preferPrimaryBalanceBuffer;
+    }
+
+    public float getPreferPrimaryBalanceBuffer() {
+        return preferPrimaryBalanceBuffer;
+    }
+}
diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/RoutingAllocation.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/RoutingAllocation.java
index bf2db57128517..fd789774f6f4f 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/allocation/RoutingAllocation.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/RoutingAllocation.java
@@ -55,6 +55,7 @@
 
 import static java.util.Collections.emptySet;
 import static java.util.Collections.unmodifiableSet;
+import static org.opensearch.node.remotestore.RemoteStoreNodeService.isMigratingToRemoteStore;
 
 /**
  * The {@link RoutingAllocation} keep the state of the current allocation
@@ -125,6 +126,9 @@ public RoutingAllocation(
         this.clusterInfo = clusterInfo;
         this.shardSizeInfo = shardSizeInfo;
         this.currentNanoTime = currentNanoTime;
+        if (isMigratingToRemoteStore(metadata)) {
+            indexMetadataUpdater.setOngoingRemoteStoreMigration(true);
+        }
     }
 
     /** returns the nano time captured at the beginning of the allocation. used to make sure all time based decisions are aligned */
@@ -267,7 +271,7 @@ public RoutingChangesObserver changes() {
      * Returns updated {@link Metadata} based on the changes that were made to the routing nodes
      */
     public Metadata updateMetadataWithRoutingChanges(RoutingTable newRoutingTable) {
-        return indexMetadataUpdater.applyChanges(metadata, newRoutingTable);
+        return indexMetadataUpdater.applyChanges(metadata, newRoutingTable, nodes());
     }
 
     /**
diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java
index 41ace0e7661fe..b2443490dd973 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/BalancedShardsAllocator.java
@@ -46,6 +46,7 @@
 import org.opensearch.cluster.routing.allocation.ConstraintTypes;
 import org.opensearch.cluster.routing.allocation.MoveDecision;
 import org.opensearch.cluster.routing.allocation.RebalanceConstraints;
+import org.opensearch.cluster.routing.allocation.RebalanceParameter;
 import org.opensearch.cluster.routing.allocation.RoutingAllocation;
 import org.opensearch.cluster.routing.allocation.ShardAllocationDecision;
 import org.opensearch.common.inject.Inject;
@@ -61,6 +62,7 @@
 import java.util.Set;
 
 import static org.opensearch.cluster.routing.allocation.ConstraintTypes.CLUSTER_PRIMARY_SHARD_BALANCE_CONSTRAINT_ID;
+import static org.opensearch.cluster.routing.allocation.ConstraintTypes.CLUSTER_PRIMARY_SHARD_REBALANCE_CONSTRAINT_ID;
 import static org.opensearch.cluster.routing.allocation.ConstraintTypes.INDEX_PRIMARY_SHARD_BALANCE_CONSTRAINT_ID;
 import static org.opensearch.cluster.routing.allocation.ConstraintTypes.INDEX_SHARD_PER_NODE_BREACH_CONSTRAINT_ID;
 
@@ -145,10 +147,29 @@ public class BalancedShardsAllocator implements ShardsAllocator {
         Property.NodeScope
     );
 
+    public static final Setting<Boolean> PREFER_PRIMARY_SHARD_REBALANCE = Setting.boolSetting(
+        "cluster.routing.allocation.rebalance.primary.enable",
+        false,
+        Property.Dynamic,
+        Property.NodeScope
+    );
+
+    public static final Setting<Float> PRIMARY_SHARD_REBALANCE_BUFFER = Setting.floatSetting(
+        "cluster.routing.allocation.rebalance.primary.buffer",
+        0.10f,
+        0.0f,
+        Property.Dynamic,
+        Property.NodeScope
+    );
+
     private volatile boolean movePrimaryFirst;
     private volatile ShardMovementStrategy shardMovementStrategy;
 
     private volatile boolean preferPrimaryShardBalance;
+    private volatile boolean preferPrimaryShardRebalance;
+    private volatile float preferPrimaryShardRebalanceBuffer;
+    private volatile float indexBalanceFactor;
+    private volatile float shardBalanceFactor;
     private volatile WeightFunction weightFunction;
     private volatile float threshold;
 
@@ -158,14 +179,21 @@ public BalancedShardsAllocator(Settings settings) {
 
     @Inject
     public BalancedShardsAllocator(Settings settings, ClusterSettings clusterSettings) {
-        setWeightFunction(INDEX_BALANCE_FACTOR_SETTING.get(settings), SHARD_BALANCE_FACTOR_SETTING.get(settings));
+        setShardBalanceFactor(SHARD_BALANCE_FACTOR_SETTING.get(settings));
+        setIndexBalanceFactor(INDEX_BALANCE_FACTOR_SETTING.get(settings));
+        setPreferPrimaryShardRebalanceBuffer(PRIMARY_SHARD_REBALANCE_BUFFER.get(settings));
+        updateWeightFunction();
         setThreshold(THRESHOLD_SETTING.get(settings));
         setPreferPrimaryShardBalance(PREFER_PRIMARY_SHARD_BALANCE.get(settings));
+        setPreferPrimaryShardRebalance(PREFER_PRIMARY_SHARD_REBALANCE.get(settings));
         setShardMovementStrategy(SHARD_MOVEMENT_STRATEGY_SETTING.get(settings));
         clusterSettings.addSettingsUpdateConsumer(PREFER_PRIMARY_SHARD_BALANCE, this::setPreferPrimaryShardBalance);
         clusterSettings.addSettingsUpdateConsumer(SHARD_MOVE_PRIMARY_FIRST_SETTING, this::setMovePrimaryFirst);
         clusterSettings.addSettingsUpdateConsumer(SHARD_MOVEMENT_STRATEGY_SETTING, this::setShardMovementStrategy);
-        clusterSettings.addSettingsUpdateConsumer(INDEX_BALANCE_FACTOR_SETTING, SHARD_BALANCE_FACTOR_SETTING, this::setWeightFunction);
+        clusterSettings.addSettingsUpdateConsumer(INDEX_BALANCE_FACTOR_SETTING, this::updateIndexBalanceFactor);
+        clusterSettings.addSettingsUpdateConsumer(SHARD_BALANCE_FACTOR_SETTING, this::updateShardBalanceFactor);
+        clusterSettings.addSettingsUpdateConsumer(PRIMARY_SHARD_REBALANCE_BUFFER, this::updatePreferPrimaryShardBalanceBuffer);
+        clusterSettings.addSettingsUpdateConsumer(PREFER_PRIMARY_SHARD_REBALANCE, this::setPreferPrimaryShardRebalance);
         clusterSettings.addSettingsUpdateConsumer(THRESHOLD_SETTING, this::setThreshold);
     }
 
@@ -190,8 +218,35 @@ private void setShardMovementStrategy(ShardMovementStrategy shardMovementStrateg
         }
     }
 
-    private void setWeightFunction(float indexBalance, float shardBalanceFactor) {
-        weightFunction = new WeightFunction(indexBalance, shardBalanceFactor);
+    private void setIndexBalanceFactor(float indexBalanceFactor) {
+        this.indexBalanceFactor = indexBalanceFactor;
+    }
+
+    private void setShardBalanceFactor(float shardBalanceFactor) {
+        this.shardBalanceFactor = shardBalanceFactor;
+    }
+
+    private void setPreferPrimaryShardRebalanceBuffer(float preferPrimaryShardRebalanceBuffer) {
+        this.preferPrimaryShardRebalanceBuffer = preferPrimaryShardRebalanceBuffer;
+    }
+
+    private void updateIndexBalanceFactor(float indexBalanceFactor) {
+        this.indexBalanceFactor = indexBalanceFactor;
+        updateWeightFunction();
+    }
+
+    private void updateShardBalanceFactor(float shardBalanceFactor) {
+        this.shardBalanceFactor = shardBalanceFactor;
+        updateWeightFunction();
+    }
+
+    private void updatePreferPrimaryShardBalanceBuffer(float preferPrimaryShardBalanceBuffer) {
+        this.preferPrimaryShardRebalanceBuffer = preferPrimaryShardBalanceBuffer;
+        updateWeightFunction();
+    }
+
+    private void updateWeightFunction() {
+        weightFunction = new WeightFunction(this.indexBalanceFactor, this.shardBalanceFactor, this.preferPrimaryShardRebalanceBuffer);
     }
 
     /**
@@ -205,6 +260,11 @@ private void setPreferPrimaryShardBalance(boolean preferPrimaryShardBalance) {
         this.weightFunction.updateRebalanceConstraint(INDEX_PRIMARY_SHARD_BALANCE_CONSTRAINT_ID, preferPrimaryShardBalance);
     }
 
+    private void setPreferPrimaryShardRebalance(boolean preferPrimaryShardRebalance) {
+        this.preferPrimaryShardRebalance = preferPrimaryShardRebalance;
+        this.weightFunction.updateRebalanceConstraint(CLUSTER_PRIMARY_SHARD_REBALANCE_CONSTRAINT_ID, preferPrimaryShardRebalance);
+    }
+
     private void setThreshold(float threshold) {
         this.threshold = threshold;
     }
@@ -221,7 +281,8 @@ public void allocate(RoutingAllocation allocation) {
             shardMovementStrategy,
             weightFunction,
             threshold,
-            preferPrimaryShardBalance
+            preferPrimaryShardBalance,
+            preferPrimaryShardRebalance
         );
         localShardsBalancer.allocateUnassigned();
         localShardsBalancer.moveShards();
@@ -242,7 +303,8 @@ public ShardAllocationDecision decideShardAllocation(final ShardRouting shard, f
             shardMovementStrategy,
             weightFunction,
             threshold,
-            preferPrimaryShardBalance
+            preferPrimaryShardBalance,
+            preferPrimaryShardRebalance
         );
         AllocateUnassignedDecision allocateUnassignedDecision = AllocateUnassignedDecision.NOT_TAKEN;
         MoveDecision moveDecision = MoveDecision.NOT_TAKEN;
@@ -348,7 +410,7 @@ static class WeightFunction {
         private AllocationConstraints constraints;
         private RebalanceConstraints rebalanceConstraints;
 
-        WeightFunction(float indexBalance, float shardBalance) {
+        WeightFunction(float indexBalance, float shardBalance, float preferPrimaryBalanceBuffer) {
             float sum = indexBalance + shardBalance;
             if (sum <= 0.0f) {
                 throw new IllegalArgumentException("Balance factors must sum to a value > 0 but was: " + sum);
@@ -357,8 +419,9 @@ static class WeightFunction {
             theta1 = indexBalance / sum;
             this.indexBalance = indexBalance;
             this.shardBalance = shardBalance;
+            RebalanceParameter rebalanceParameter = new RebalanceParameter(preferPrimaryBalanceBuffer);
             this.constraints = new AllocationConstraints();
-            this.rebalanceConstraints = new RebalanceConstraints();
+            this.rebalanceConstraints = new RebalanceConstraints(rebalanceParameter);
             // Enable index shard per node breach constraint
             updateAllocationConstraint(INDEX_SHARD_PER_NODE_BREACH_CONSTRAINT_ID, true);
         }
@@ -495,7 +558,7 @@ public Balancer(
             float threshold,
             boolean preferPrimaryBalance
         ) {
-            super(logger, allocation, shardMovementStrategy, weight, threshold, preferPrimaryBalance);
+            super(logger, allocation, shardMovementStrategy, weight, threshold, preferPrimaryBalance, false);
         }
     }
 
diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/LocalShardsBalancer.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/LocalShardsBalancer.java
index 45f64a5b29b04..ec25d041bda43 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/LocalShardsBalancer.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/allocator/LocalShardsBalancer.java
@@ -61,6 +61,7 @@ public class LocalShardsBalancer extends ShardsBalancer {
     private final ShardMovementStrategy shardMovementStrategy;
 
     private final boolean preferPrimaryBalance;
+    private final boolean preferPrimaryRebalance;
     private final BalancedShardsAllocator.WeightFunction weight;
 
     private final float threshold;
@@ -76,7 +77,8 @@ public LocalShardsBalancer(
         ShardMovementStrategy shardMovementStrategy,
         BalancedShardsAllocator.WeightFunction weight,
         float threshold,
-        boolean preferPrimaryBalance
+        boolean preferPrimaryBalance,
+        boolean preferPrimaryRebalance
     ) {
         this.logger = logger;
         this.allocation = allocation;
@@ -91,6 +93,7 @@ public LocalShardsBalancer(
         sorter = newNodeSorter();
         inEligibleTargetNode = new HashSet<>();
         this.preferPrimaryBalance = preferPrimaryBalance;
+        this.preferPrimaryRebalance = preferPrimaryRebalance;
         this.shardMovementStrategy = shardMovementStrategy;
     }
 
@@ -995,13 +998,18 @@ private boolean tryRelocateShard(BalancedShardsAllocator.ModelNode minNode, Bala
                     continue;
                 }
                 // This is a safety net which prevents un-necessary primary shard relocations from maxNode to minNode when
-                // doing such relocation wouldn't help in primary balance.
+                // doing such relocation wouldn't help in primary balance. The condition won't be applicable when we enable node level
+                // primary rebalance
                 if (preferPrimaryBalance == true
+                    && preferPrimaryRebalance == false
                     && shard.primary()
                     && maxNode.numPrimaryShards(shard.getIndexName()) - minNode.numPrimaryShards(shard.getIndexName()) < 2) {
                     continue;
                 }
-
+                // Relax the above condition to per node to allow rebalancing to attain global balance
+                if (preferPrimaryRebalance == true && shard.primary() && maxNode.numPrimaryShards() - minNode.numPrimaryShards() < 2) {
+                    continue;
+                }
                 final Decision decision = new Decision.Multi().add(allocationDecision).add(rebalanceDecision);
                 maxNode.removeShard(shard);
                 long shardSize = allocation.clusterInfo().getShardSize(shard, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE);
diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/FilterAllocationDecider.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/FilterAllocationDecider.java
index af4b2c61a95b1..d3200c1bc9d75 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/FilterAllocationDecider.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/FilterAllocationDecider.java
@@ -38,11 +38,13 @@
 import org.opensearch.cluster.routing.RecoverySource;
 import org.opensearch.cluster.routing.RoutingNode;
 import org.opensearch.cluster.routing.ShardRouting;
+import org.opensearch.cluster.routing.UnassignedInfo;
 import org.opensearch.cluster.routing.allocation.RoutingAllocation;
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Setting.Property;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.node.remotestore.RemoteStoreNodeService;
 
 import java.util.Map;
 
@@ -102,14 +104,32 @@ public class FilterAllocationDecider extends AllocationDecider {
     private volatile DiscoveryNodeFilters clusterRequireFilters;
     private volatile DiscoveryNodeFilters clusterIncludeFilters;
     private volatile DiscoveryNodeFilters clusterExcludeFilters;
+    private volatile RemoteStoreNodeService.Direction migrationDirection;
+    private volatile RemoteStoreNodeService.CompatibilityMode compatibilityMode;
 
     public FilterAllocationDecider(Settings settings, ClusterSettings clusterSettings) {
         setClusterRequireFilters(CLUSTER_ROUTING_REQUIRE_GROUP_SETTING.getAsMap(settings));
         setClusterExcludeFilters(CLUSTER_ROUTING_EXCLUDE_GROUP_SETTING.getAsMap(settings));
         setClusterIncludeFilters(CLUSTER_ROUTING_INCLUDE_GROUP_SETTING.getAsMap(settings));
+        this.migrationDirection = RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING.get(settings);
+        this.compatibilityMode = RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING.get(settings);
+
         clusterSettings.addAffixMapUpdateConsumer(CLUSTER_ROUTING_REQUIRE_GROUP_SETTING, this::setClusterRequireFilters, (a, b) -> {});
         clusterSettings.addAffixMapUpdateConsumer(CLUSTER_ROUTING_EXCLUDE_GROUP_SETTING, this::setClusterExcludeFilters, (a, b) -> {});
         clusterSettings.addAffixMapUpdateConsumer(CLUSTER_ROUTING_INCLUDE_GROUP_SETTING, this::setClusterIncludeFilters, (a, b) -> {});
+        clusterSettings.addSettingsUpdateConsumer(RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING, this::setMigrationDirection);
+        clusterSettings.addSettingsUpdateConsumer(
+            RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING,
+            this::setCompatibilityMode
+        );
+    }
+
+    private void setMigrationDirection(RemoteStoreNodeService.Direction migrationDirection) {
+        this.migrationDirection = migrationDirection;
+    }
+
+    private void setCompatibilityMode(RemoteStoreNodeService.CompatibilityMode compatibilityMode) {
+        this.compatibilityMode = compatibilityMode;
     }
 
     @Override
@@ -127,10 +147,28 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
                     "initial allocation of the shrunken index is only allowed on nodes [%s] that hold a copy of every shard in the index";
                 return allocation.decision(Decision.NO, NAME, explanation, initialRecoveryFilters);
             }
+
+            Decision decision = isRemoteStoreMigrationReplicaDecision(shardRouting, allocation);
+            if (decision != null) return decision;
         }
         return shouldFilter(shardRouting, node.node(), allocation);
     }
 
+    public Decision isRemoteStoreMigrationReplicaDecision(ShardRouting shardRouting, RoutingAllocation allocation) {
+        assert shardRouting.unassigned();
+        boolean primaryOnRemote = RemoteStoreMigrationAllocationDecider.isPrimaryOnRemote(shardRouting.shardId(), allocation);
+        if (shardRouting.primary() == false
+            && shardRouting.unassignedInfo().getReason() != UnassignedInfo.Reason.INDEX_CREATED
+            && (compatibilityMode.equals(RemoteStoreNodeService.CompatibilityMode.MIXED))
+            && (migrationDirection.equals(RemoteStoreNodeService.Direction.REMOTE_STORE))
+            && primaryOnRemote == false) {
+            String explanation =
+                "in  remote store migration, allocation filters are not applicable for replica copies whose primary is on doc rep node";
+            return allocation.decision(Decision.YES, NAME, explanation);
+        }
+        return null;
+    }
+
     @Override
     public Decision canAllocate(IndexMetadata indexMetadata, RoutingNode node, RoutingAllocation allocation) {
         return shouldFilter(indexMetadata, node.node(), allocation);
diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/RemoteStoreMigrationAllocationDecider.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/RemoteStoreMigrationAllocationDecider.java
index 27ebe5390ea6d..4fc5fff805663 100644
--- a/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/RemoteStoreMigrationAllocationDecider.java
+++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/decider/RemoteStoreMigrationAllocationDecider.java
@@ -39,6 +39,7 @@
 import org.opensearch.cluster.routing.allocation.RoutingAllocation;
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.core.index.shard.ShardId;
 import org.opensearch.node.remotestore.RemoteStoreNodeService;
 import org.opensearch.node.remotestore.RemoteStoreNodeService.CompatibilityMode;
 import org.opensearch.node.remotestore.RemoteStoreNodeService.Direction;
@@ -60,9 +61,8 @@ public class RemoteStoreMigrationAllocationDecider extends AllocationDecider {
 
     public static final String NAME = "remote_store_migration";
 
-    private Direction migrationDirection;
-    private CompatibilityMode compatibilityMode;
-    private boolean remoteStoreBackedIndex;
+    volatile private Direction migrationDirection;
+    volatile private CompatibilityMode compatibilityMode;
 
     public RemoteStoreMigrationAllocationDecider(Settings settings, ClusterSettings clusterSettings) {
         this.migrationDirection = RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING.get(settings);
@@ -95,34 +95,38 @@ public Decision canAllocate(ShardRouting shardRouting, RoutingNode node, Routing
             );
         }
 
-        if (migrationDirection.equals(Direction.REMOTE_STORE) == false) {
-            // docrep migration direction is currently not supported
+        IndexMetadata indexMetadata = allocation.metadata().getIndexSafe(shardRouting.index());
+        boolean remoteSettingsBackedIndex = IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING.get(indexMetadata.getSettings());
+
+        if (migrationDirection.equals(Direction.NONE)) {
+            // remote backed indices on docrep nodes and non remote backed indices on remote nodes are not allowed
+            boolean isNoDecision = remoteSettingsBackedIndex ^ targetNode.isRemoteStoreNode();
+            String reason = String.format(Locale.ROOT, " for %sremote store backed index", remoteSettingsBackedIndex ? "" : "non ");
             return allocation.decision(
-                Decision.YES,
+                isNoDecision ? Decision.NO : Decision.YES,
                 NAME,
-                getDecisionDetails(true, shardRouting, targetNode, " for non remote_store direction")
-            );
-        }
-
-        // check for remote store backed indices
-        IndexMetadata indexMetadata = allocation.metadata().getIndexSafe(shardRouting.index());
-        if (IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING.exists(indexMetadata.getSettings())) {
-            remoteStoreBackedIndex = IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING.get(indexMetadata.getSettings());
-        }
-        if (remoteStoreBackedIndex && targetNode.isRemoteStoreNode() == false) {
-            // allocations and relocations must be to a remote node
-            String reason = String.format(
-                Locale.ROOT,
-                " because a remote store backed index's shard copy can only be %s to a remote node",
-                ((shardRouting.assignedToNode() == false) ? "allocated" : "relocated")
+                getDecisionDetails(!isNoDecision, shardRouting, targetNode, reason)
             );
-            return allocation.decision(Decision.NO, NAME, getDecisionDetails(false, shardRouting, targetNode, reason));
-        }
+        } else if (migrationDirection.equals(Direction.DOCREP)) {
+            // docrep migration direction is currently not supported
+            return allocation.decision(Decision.YES, NAME, getDecisionDetails(true, shardRouting, targetNode, " for DOCREP direction"));
+        } else {
+            // check for remote store backed indices
+            if (remoteSettingsBackedIndex && targetNode.isRemoteStoreNode() == false) {
+                // allocations and relocations must be to a remote node
+                String reason = String.format(
+                    Locale.ROOT,
+                    " because a remote store backed index's shard copy can only be %s to a remote node",
+                    ((shardRouting.assignedToNode() == false) ? "allocated" : "relocated")
+                );
+                return allocation.decision(Decision.NO, NAME, getDecisionDetails(false, shardRouting, targetNode, reason));
+            }
 
-        if (shardRouting.primary()) {
-            return primaryShardDecision(shardRouting, targetNode, allocation);
+            if (shardRouting.primary()) {
+                return primaryShardDecision(shardRouting, targetNode, allocation);
+            }
+            return replicaShardDecision(shardRouting, targetNode, allocation);
         }
-        return replicaShardDecision(shardRouting, targetNode, allocation);
     }
 
     // handle scenarios for allocation of a new shard's primary copy
@@ -133,15 +137,20 @@ private Decision primaryShardDecision(ShardRouting primaryShardRouting, Discover
         return allocation.decision(Decision.YES, NAME, getDecisionDetails(true, primaryShardRouting, targetNode, ""));
     }
 
+    // Checks if primary shard is on a remote node.
+    static boolean isPrimaryOnRemote(ShardId shardId, RoutingAllocation allocation) {
+        ShardRouting primaryShardRouting = allocation.routingNodes().activePrimary(shardId);
+        if (primaryShardRouting != null) {
+            DiscoveryNode primaryShardNode = allocation.nodes().getNodes().get(primaryShardRouting.currentNodeId());
+            return primaryShardNode.isRemoteStoreNode();
+        }
+        return false;
+    }
+
     private Decision replicaShardDecision(ShardRouting replicaShardRouting, DiscoveryNode targetNode, RoutingAllocation allocation) {
         if (targetNode.isRemoteStoreNode()) {
-            ShardRouting primaryShardRouting = allocation.routingNodes().activePrimary(replicaShardRouting.shardId());
-            boolean primaryHasMigratedToRemote = false;
-            if (primaryShardRouting != null) {
-                DiscoveryNode primaryShardNode = allocation.nodes().getNodes().get(primaryShardRouting.currentNodeId());
-                primaryHasMigratedToRemote = primaryShardNode.isRemoteStoreNode();
-            }
-            if (primaryHasMigratedToRemote == false) {
+            boolean primaryOnRemote = RemoteStoreMigrationAllocationDecider.isPrimaryOnRemote(replicaShardRouting.shardId(), allocation);
+            if (primaryOnRemote == false) {
                 return allocation.decision(
                     Decision.NO,
                     NAME,
diff --git a/server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java b/server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java
index a2b7a92c58c3f..341cc22f80770 100644
--- a/server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java
+++ b/server/src/main/java/org/opensearch/cluster/service/ClusterApplierService.java
@@ -36,6 +36,7 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.opensearch.cluster.ClusterChangedEvent;
+import org.opensearch.cluster.ClusterManagerMetrics;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.ClusterStateApplier;
 import org.opensearch.cluster.ClusterStateListener;
@@ -61,8 +62,6 @@
 import org.opensearch.common.util.concurrent.PrioritizedOpenSearchThreadPoolExecutor;
 import org.opensearch.common.util.concurrent.ThreadContext;
 import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException;
-import org.opensearch.telemetry.metrics.Histogram;
-import org.opensearch.telemetry.metrics.MetricsRegistry;
 import org.opensearch.telemetry.metrics.tags.Tags;
 import org.opensearch.threadpool.Scheduler;
 import org.opensearch.threadpool.ThreadPool;
@@ -71,6 +70,7 @@
 import java.util.Collection;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.CountDownLatch;
@@ -123,17 +123,14 @@ public class ClusterApplierService extends AbstractLifecycleComponent implements
     private final String nodeName;
 
     private NodeConnectionsService nodeConnectionsService;
-
-    private Histogram clusterStateAppliersHistogram;
-
-    private Histogram clusterStateListenersHistogram;
+    private final ClusterManagerMetrics clusterManagerMetrics;
 
     public ClusterApplierService(
         String nodeName,
         Settings settings,
         ClusterSettings clusterSettings,
         ThreadPool threadPool,
-        MetricsRegistry metricsRegistry
+        ClusterManagerMetrics clusterManagerMetrics
     ) {
         this.clusterSettings = clusterSettings;
         this.threadPool = threadPool;
@@ -145,20 +142,7 @@ public ClusterApplierService(
             CLUSTER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING,
             this::setSlowTaskLoggingThreshold
         );
-        initializeMetrics(metricsRegistry);
-    }
-
-    private void initializeMetrics(MetricsRegistry metricsRegistry) {
-        this.clusterStateAppliersHistogram = metricsRegistry.createHistogram(
-            "cluster.state.appliers.latency",
-            "Histogram for tracking the latency of cluster state appliers",
-            "ms"
-        );
-        this.clusterStateListenersHistogram = metricsRegistry.createHistogram(
-            "cluster.state.listeners.latency",
-            "Histogram for tracking the latency of cluster state listeners",
-            "ms"
-        );
+        this.clusterManagerMetrics = clusterManagerMetrics;
     }
 
     private void setSlowTaskLoggingThreshold(TimeValue slowTaskLoggingThreshold) {
@@ -634,9 +618,10 @@ private void callClusterStateAppliers(
             try (TimingHandle ignored = stopWatch.timing("running applier [" + applier + "]")) {
                 long applierStartTimeNS = System.nanoTime();
                 applier.applyClusterState(clusterChangedEvent);
-                clusterStateAppliersHistogram.record(
+                ClusterManagerMetrics.recordLatency(
+                    clusterManagerMetrics.clusterStateAppliersHistogram,
                     (double) Math.max(0, TimeValue.nsecToMSec(System.nanoTime() - applierStartTimeNS)),
-                    Tags.create().addTag("Operation", applier.getClass().getSimpleName())
+                    Optional.of(Tags.create().addTag("Operation", applier.getClass().getSimpleName()))
                 );
             }
         }
@@ -658,9 +643,10 @@ private void callClusterStateListener(
                 try (TimingHandle ignored = stopWatch.timing("notifying listener [" + listener + "]")) {
                     long listenerStartTimeNS = System.nanoTime();
                     listener.clusterChanged(clusterChangedEvent);
-                    clusterStateListenersHistogram.record(
+                    ClusterManagerMetrics.recordLatency(
+                        clusterManagerMetrics.clusterStateListenersHistogram,
                         (double) Math.max(0, TimeValue.nsecToMSec(System.nanoTime() - listenerStartTimeNS)),
-                        Tags.create().addTag("Operation", listener.getClass().getSimpleName())
+                        Optional.of(Tags.create().addTag("Operation", listener.getClass().getSimpleName()))
                     );
                 }
             } catch (Exception ex) {
diff --git a/server/src/main/java/org/opensearch/cluster/service/ClusterManagerService.java b/server/src/main/java/org/opensearch/cluster/service/ClusterManagerService.java
index 7d5ceb157e5aa..eaedb36a59f1e 100644
--- a/server/src/main/java/org/opensearch/cluster/service/ClusterManagerService.java
+++ b/server/src/main/java/org/opensearch/cluster/service/ClusterManagerService.java
@@ -8,10 +8,10 @@
 
 package org.opensearch.cluster.service;
 
+import org.opensearch.cluster.ClusterManagerMetrics;
 import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Settings;
-import org.opensearch.telemetry.metrics.MetricsRegistry;
 import org.opensearch.threadpool.ThreadPool;
 
 /**
@@ -25,8 +25,8 @@ public ClusterManagerService(
         Settings settings,
         ClusterSettings clusterSettings,
         ThreadPool threadPool,
-        MetricsRegistry metricsRegistry
+        ClusterManagerMetrics clusterManagerMetrics
     ) {
-        super(settings, clusterSettings, threadPool, metricsRegistry);
+        super(settings, clusterSettings, threadPool, clusterManagerMetrics);
     }
 }
diff --git a/server/src/main/java/org/opensearch/cluster/service/ClusterService.java b/server/src/main/java/org/opensearch/cluster/service/ClusterService.java
index 50566bc2743ac..fa61375e85c25 100644
--- a/server/src/main/java/org/opensearch/cluster/service/ClusterService.java
+++ b/server/src/main/java/org/opensearch/cluster/service/ClusterService.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.cluster.service;
 
+import org.opensearch.cluster.ClusterManagerMetrics;
 import org.opensearch.cluster.ClusterName;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.ClusterStateApplier;
@@ -53,7 +54,6 @@
 import org.opensearch.common.settings.Settings;
 import org.opensearch.index.IndexingPressureService;
 import org.opensearch.node.Node;
-import org.opensearch.telemetry.metrics.MetricsRegistry;
 import org.opensearch.threadpool.ThreadPool;
 
 import java.util.Collections;
@@ -92,12 +92,17 @@ public class ClusterService extends AbstractLifecycleComponent {
 
     private IndexingPressureService indexingPressureService;
 
-    public ClusterService(Settings settings, ClusterSettings clusterSettings, ThreadPool threadPool, MetricsRegistry metricsRegistry) {
+    public ClusterService(
+        Settings settings,
+        ClusterSettings clusterSettings,
+        ThreadPool threadPool,
+        ClusterManagerMetrics clusterManagerMetrics
+    ) {
         this(
             settings,
             clusterSettings,
-            new ClusterManagerService(settings, clusterSettings, threadPool, metricsRegistry),
-            new ClusterApplierService(Node.NODE_NAME_SETTING.get(settings), settings, clusterSettings, threadPool, metricsRegistry)
+            new ClusterManagerService(settings, clusterSettings, threadPool, clusterManagerMetrics),
+            new ClusterApplierService(Node.NODE_NAME_SETTING.get(settings), settings, clusterSettings, threadPool, clusterManagerMetrics)
         );
     }
 
diff --git a/server/src/main/java/org/opensearch/cluster/service/MasterService.java b/server/src/main/java/org/opensearch/cluster/service/MasterService.java
index 7ecb08678fede..214ee83cec9b7 100644
--- a/server/src/main/java/org/opensearch/cluster/service/MasterService.java
+++ b/server/src/main/java/org/opensearch/cluster/service/MasterService.java
@@ -39,6 +39,7 @@
 import org.opensearch.action.support.PlainActionFuture;
 import org.opensearch.cluster.AckedClusterStateTaskListener;
 import org.opensearch.cluster.ClusterChangedEvent;
+import org.opensearch.cluster.ClusterManagerMetrics;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.ClusterState.Builder;
 import org.opensearch.cluster.ClusterStateTaskConfig;
@@ -70,8 +71,6 @@
 import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException;
 import org.opensearch.discovery.Discovery;
 import org.opensearch.node.Node;
-import org.opensearch.telemetry.metrics.Histogram;
-import org.opensearch.telemetry.metrics.MetricsRegistry;
 import org.opensearch.telemetry.metrics.tags.Tags;
 import org.opensearch.threadpool.Scheduler;
 import org.opensearch.threadpool.ThreadPool;
@@ -82,6 +81,7 @@
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
@@ -138,10 +138,14 @@ public class MasterService extends AbstractLifecycleComponent {
     protected final ClusterManagerTaskThrottler clusterManagerTaskThrottler;
     private final ClusterManagerThrottlingStats throttlingStats;
     private final ClusterStateStats stateStats;
-    private Histogram clusterStateComputeHistogram;
-    private Histogram clusterStatePublishHistogram;
+    private final ClusterManagerMetrics clusterManagerMetrics;
 
-    public MasterService(Settings settings, ClusterSettings clusterSettings, ThreadPool threadPool, MetricsRegistry metricsRegistry) {
+    public MasterService(
+        Settings settings,
+        ClusterSettings clusterSettings,
+        ThreadPool threadPool,
+        ClusterManagerMetrics clusterManagerMetrics
+    ) {
         this.nodeName = Objects.requireNonNull(Node.NODE_NAME_SETTING.get(settings));
 
         this.slowTaskLoggingThreshold = CLUSTER_MANAGER_SERVICE_SLOW_TASK_LOGGING_THRESHOLD_SETTING.get(settings);
@@ -159,20 +163,7 @@ public MasterService(Settings settings, ClusterSettings clusterSettings, ThreadP
         );
         this.stateStats = new ClusterStateStats();
         this.threadPool = threadPool;
-        initializeMetrics(metricsRegistry);
-    }
-
-    private void initializeMetrics(MetricsRegistry metricsRegistry) {
-        this.clusterStateComputeHistogram = metricsRegistry.createHistogram(
-            "cluster.state.new.compute.latency",
-            "Histogram for recording time taken to compute new cluster state",
-            "ms"
-        );
-        this.clusterStatePublishHistogram = metricsRegistry.createHistogram(
-            "cluster.state.publish.success.latency",
-            "Histogram for recording time taken to publish a new cluster state",
-            "ms"
-        );
+        this.clusterManagerMetrics = clusterManagerMetrics;
     }
 
     private void setSlowTaskLoggingThreshold(TimeValue slowTaskLoggingThreshold) {
@@ -321,9 +312,11 @@ private void runTasks(TaskInputs taskInputs) {
         taskOutputs.notifyFailedTasks();
         final TimeValue computationTime = getTimeSince(computationStartTime);
         logExecutionTime(computationTime, "compute cluster state update", summary);
-        clusterStateComputeHistogram.record(
-            computationTime.getMillis(),
-            Tags.create().addTag("Operation", taskInputs.executor.getClass().getSimpleName())
+
+        ClusterManagerMetrics.recordLatency(
+            clusterManagerMetrics.clusterStateComputeHistogram,
+            (double) computationTime.getMillis(),
+            Optional.of(Tags.create().addTag("Operation", taskInputs.executor.getClass().getSimpleName()))
         );
 
         if (taskOutputs.clusterStateUnchanged()) {
@@ -384,7 +377,7 @@ protected boolean blockingAllowed() {
             final long durationMillis = getTimeSince(startTimeNanos).millis();
             stateStats.stateUpdateTook(durationMillis);
             stateStats.stateUpdated();
-            clusterStatePublishHistogram.record(durationMillis);
+            ClusterManagerMetrics.recordLatency(clusterManagerMetrics.clusterStatePublishHistogram, (double) durationMillis);
         } catch (Exception e) {
             stateStats.stateUpdateFailed();
             onPublicationFailed(clusterChangedEvent, taskOutputs, startTimeNanos, e);
diff --git a/server/src/main/java/org/opensearch/common/blobstore/BlobContainer.java b/server/src/main/java/org/opensearch/common/blobstore/BlobContainer.java
index 2e25a532b5abf..4f5f8d4b1ef5f 100644
--- a/server/src/main/java/org/opensearch/common/blobstore/BlobContainer.java
+++ b/server/src/main/java/org/opensearch/common/blobstore/BlobContainer.java
@@ -32,6 +32,8 @@
 
 package org.opensearch.common.blobstore;
 
+import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.ExperimentalApi;
 import org.opensearch.core.action.ActionListener;
 
 import java.io.IOException;
@@ -77,6 +79,20 @@ public interface BlobContainer {
      */
     InputStream readBlob(String blobName) throws IOException;
 
+    /**
+     * Creates a new {@link FetchBlobResult} for the given blob name.
+     *
+     * @param   blobName
+     *          The name of the blob to get an {@link InputStream} for.
+     * @return  The {@link FetchBlobResult} of the blob.
+     * @throws  NoSuchFileException if the blob does not exist
+     * @throws  IOException if the blob can not be read.
+     */
+    @ExperimentalApi
+    default FetchBlobResult readBlobWithMetadata(String blobName) throws IOException {
+        throw new UnsupportedOperationException("readBlobWithMetadata is not implemented yet");
+    };
+
     /**
      * Creates a new {@link InputStream} that can be used to read the given blob starting from
      * a specific {@code position} in the blob. The {@code length} is an indication of the
@@ -128,6 +144,36 @@ default long readBlobPreferredLength() {
      */
     void writeBlob(String blobName, InputStream inputStream, long blobSize, boolean failIfAlreadyExists) throws IOException;
 
+    /**
+     * Reads blob content from the input stream and writes it to the container in a new blob with the given name, and metadata.
+     * This method assumes the container does not already contain a blob of the same blobName. If a blob by the
+     * same name already exists, the operation will fail and an {@link IOException} will be thrown.
+     *
+     * @param   blobName
+     *          The name of the blob to write the contents of the input stream to.
+     * @param   inputStream
+     *          The input stream from which to retrieve the bytes to write to the blob.
+     * @param   metadata
+     *          The metadata to be associate with the blob upload.
+     * @param   blobSize
+     *          The size of the blob to be written, in bytes.  It is implementation dependent whether
+     *          this value is used in writing the blob to the repository.
+     * @param   failIfAlreadyExists
+     *          whether to throw a FileAlreadyExistsException if the given blob already exists
+     * @throws  FileAlreadyExistsException if failIfAlreadyExists is true and a blob by the same name already exists
+     * @throws  IOException if the input stream could not be read, or the target blob could not be written to.
+     */
+    @ExperimentalApi
+    default void writeBlobWithMetadata(
+        String blobName,
+        InputStream inputStream,
+        long blobSize,
+        boolean failIfAlreadyExists,
+        @Nullable Map<String, String> metadata
+    ) throws IOException {
+        throw new UnsupportedOperationException("writeBlobWithMetadata is not implemented yet");
+    };
+
     /**
      * Reads blob content from the input stream and writes it to the container in a new blob with the given name,
      * using an atomic write operation if the implementation supports it.
@@ -149,6 +195,38 @@ default long readBlobPreferredLength() {
      */
     void writeBlobAtomic(String blobName, InputStream inputStream, long blobSize, boolean failIfAlreadyExists) throws IOException;
 
+    /**
+     * Reads blob content from the input stream and writes it to the container in a new blob with the given name, and metadata
+     * using an atomic write operation if the implementation supports it.
+     * <p>
+     * This method assumes the container does not already contain a blob of the same blobName. If a blob by the
+     * same name already exists, the operation will fail and an {@link IOException} will be thrown.
+     *
+     * @param   blobName
+     *          The name of the blob to write the contents of the input stream to.
+     * @param   inputStream
+     *          The input stream from which to retrieve the bytes to write to the blob.
+     * @param   metadata
+     *          The metadata to be associate with the blob upload.
+     * @param   blobSize
+     *          The size of the blob to be written, in bytes.  It is implementation dependent whether
+     *          this value is used in writing the blob to the repository.
+     * @param   failIfAlreadyExists
+     *          whether to throw a FileAlreadyExistsException if the given blob already exists
+     * @throws  FileAlreadyExistsException if failIfAlreadyExists is true and a blob by the same name already exists
+     * @throws  IOException if the input stream could not be read, or the target blob could not be written to.
+     */
+    @ExperimentalApi
+    default void writeBlobAtomicWithMetadata(
+        String blobName,
+        InputStream inputStream,
+        @Nullable Map<String, String> metadata,
+        long blobSize,
+        boolean failIfAlreadyExists
+    ) throws IOException {
+        throw new UnsupportedOperationException("writeBlobAtomicWithMetadata is not implemented yet");
+    };
+
     /**
      * Deletes this container and all its contents from the repository.
      *
diff --git a/server/src/main/java/org/opensearch/common/blobstore/BlobPath.java b/server/src/main/java/org/opensearch/common/blobstore/BlobPath.java
index c54536e9c46e2..6f3e8be7c28b8 100644
--- a/server/src/main/java/org/opensearch/common/blobstore/BlobPath.java
+++ b/server/src/main/java/org/opensearch/common/blobstore/BlobPath.java
@@ -33,6 +33,7 @@
 package org.opensearch.common.blobstore;
 
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 
 import java.util.ArrayList;
 import java.util.Collections;
@@ -42,8 +43,9 @@
 /**
  * The list of paths where a blob can reside.  The contents of the paths are dependent upon the implementation of {@link BlobContainer}.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class BlobPath implements Iterable<String> {
 
     private static final String SEPARATOR = "/";
@@ -77,6 +79,15 @@ public BlobPath add(String path) {
         return new BlobPath(Collections.unmodifiableList(paths));
     }
 
+    /**
+     * Add additional level of paths to the existing path and returns new {@link BlobPath} with the updated paths.
+     */
+    public BlobPath add(Iterable<String> paths) {
+        List<String> updatedPaths = new ArrayList<>(this.paths);
+        paths.iterator().forEachRemaining(updatedPaths::add);
+        return new BlobPath(Collections.unmodifiableList(updatedPaths));
+    }
+
     public String buildAsString() {
         String p = String.join(SEPARATOR, paths);
         if (p.isEmpty() || p.endsWith(SEPARATOR)) {
diff --git a/server/src/main/java/org/opensearch/common/blobstore/BlobStore.java b/server/src/main/java/org/opensearch/common/blobstore/BlobStore.java
index 0f6646d37f950..8ce8ec8e01abe 100644
--- a/server/src/main/java/org/opensearch/common/blobstore/BlobStore.java
+++ b/server/src/main/java/org/opensearch/common/blobstore/BlobStore.java
@@ -75,6 +75,7 @@ default void reload(RepositoryMetadata repositoryMetadata) {}
      * Metrics for BlobStore interactions
      */
     enum Metric {
+        GENERIC_STATS("generic_stats"),
         REQUEST_SUCCESS("request_success_total"),
         REQUEST_FAILURE("request_failures_total"),
         REQUEST_LATENCY("request_time_in_millis"),
diff --git a/server/src/main/java/org/opensearch/common/blobstore/FetchBlobResult.java b/server/src/main/java/org/opensearch/common/blobstore/FetchBlobResult.java
new file mode 100644
index 0000000000000..55aca771b586c
--- /dev/null
+++ b/server/src/main/java/org/opensearch/common/blobstore/FetchBlobResult.java
@@ -0,0 +1,48 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.blobstore;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+
+import java.io.InputStream;
+import java.util.Map;
+
+/**
+ * Represents the response from a blob download operation, containing both the
+ * input stream of the blob content and the associated metadata.
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public class FetchBlobResult {
+
+    /**
+     * Downloaded blob InputStream
+     */
+    private final InputStream inputStream;
+
+    /**
+     * Metadata of the downloaded blob
+     */
+    private final Map<String, String> metadata;
+
+    public InputStream getInputStream() {
+        return inputStream;
+    }
+
+    public Map<String, String> getMetadata() {
+        return metadata;
+    }
+
+    public FetchBlobResult(InputStream inputStream, Map<String, String> metadata) {
+        this.inputStream = inputStream;
+        this.metadata = metadata;
+    }
+
+}
diff --git a/server/src/main/java/org/opensearch/common/blobstore/stream/read/ReadContext.java b/server/src/main/java/org/opensearch/common/blobstore/stream/read/ReadContext.java
index 1264551401b4c..36ee46c0fc2c8 100644
--- a/server/src/main/java/org/opensearch/common/blobstore/stream/read/ReadContext.java
+++ b/server/src/main/java/org/opensearch/common/blobstore/stream/read/ReadContext.java
@@ -12,6 +12,7 @@
 import org.opensearch.common.io.InputStreamContainer;
 
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.CompletableFuture;
 import java.util.function.Supplier;
 
@@ -25,23 +26,30 @@ public class ReadContext {
     private final long blobSize;
     private final List<StreamPartCreator> asyncPartStreams;
     private final String blobChecksum;
+    private final Map<String, String> metadata;
 
-    public ReadContext(long blobSize, List<StreamPartCreator> asyncPartStreams, String blobChecksum) {
+    private ReadContext(long blobSize, List<StreamPartCreator> asyncPartStreams, String blobChecksum, Map<String, String> metadata) {
         this.blobSize = blobSize;
         this.asyncPartStreams = asyncPartStreams;
         this.blobChecksum = blobChecksum;
+        this.metadata = metadata;
     }
 
     public ReadContext(ReadContext readContext) {
         this.blobSize = readContext.blobSize;
         this.asyncPartStreams = readContext.asyncPartStreams;
         this.blobChecksum = readContext.blobChecksum;
+        this.metadata = readContext.metadata;
     }
 
     public String getBlobChecksum() {
         return blobChecksum;
     }
 
+    public Map<String, String> getMetadata() {
+        return metadata;
+    }
+
     public int getNumberOfParts() {
         return asyncPartStreams.size();
     }
@@ -64,7 +72,7 @@ public List<StreamPartCreator> getPartStreams() {
     @ExperimentalApi
     public interface StreamPartCreator extends Supplier<CompletableFuture<InputStreamContainer>> {
         /**
-         * Kicks off a async process to start streaming.
+         * Kicks off an async process to start streaming.
          *
          * @return When the returned future is completed, streaming has
          * just begun. Clients must fully consume the resulting stream.
@@ -72,4 +80,36 @@ public interface StreamPartCreator extends Supplier<CompletableFuture<InputStrea
         @Override
         CompletableFuture<InputStreamContainer> get();
     }
+
+    /**
+     * Builder for {@link ReadContext}.
+     *
+     * @opensearch.experimental
+     */
+    public static class Builder {
+        private final long blobSize;
+        private final List<StreamPartCreator> asyncPartStreams;
+        private String blobChecksum;
+        private Map<String, String> metadata;
+
+        public Builder(long blobSize, List<StreamPartCreator> asyncPartStreams) {
+            this.blobSize = blobSize;
+            this.asyncPartStreams = asyncPartStreams;
+        }
+
+        public Builder blobChecksum(String blobChecksum) {
+            this.blobChecksum = blobChecksum;
+            return this;
+        }
+
+        public Builder metadata(Map<String, String> metadata) {
+            this.metadata = metadata;
+            return this;
+        }
+
+        public ReadContext build() {
+            return new ReadContext(blobSize, asyncPartStreams, blobChecksum, metadata);
+        }
+
+    }
 }
diff --git a/server/src/main/java/org/opensearch/common/blobstore/stream/write/WriteContext.java b/server/src/main/java/org/opensearch/common/blobstore/stream/write/WriteContext.java
index e74462f82400d..9a7fc24f7e484 100644
--- a/server/src/main/java/org/opensearch/common/blobstore/stream/write/WriteContext.java
+++ b/server/src/main/java/org/opensearch/common/blobstore/stream/write/WriteContext.java
@@ -13,6 +13,7 @@
 import org.opensearch.common.StreamContext;
 
 import java.io.IOException;
+import java.util.Map;
 
 /**
  * WriteContext is used to encapsulate all data needed by <code>BlobContainer#writeStreams</code>
@@ -29,6 +30,7 @@ public class WriteContext {
     private final CheckedConsumer<Boolean, IOException> uploadFinalizer;
     private final boolean doRemoteDataIntegrityCheck;
     private final Long expectedChecksum;
+    private final Map<String, String> metadata;
 
     /**
      * Construct a new WriteContext object
@@ -41,7 +43,7 @@ public class WriteContext {
      * @param doRemoteDataIntegrityCheck A boolean to inform vendor plugins whether remote data integrity checks need to be done
      * @param expectedChecksum           This parameter expected only when the vendor plugin is expected to do server side data integrity verification
      */
-    public WriteContext(
+    private WriteContext(
         String fileName,
         StreamContextSupplier streamContextSupplier,
         long fileSize,
@@ -49,7 +51,8 @@ public WriteContext(
         WritePriority writePriority,
         CheckedConsumer<Boolean, IOException> uploadFinalizer,
         boolean doRemoteDataIntegrityCheck,
-        @Nullable Long expectedChecksum
+        @Nullable Long expectedChecksum,
+        @Nullable Map<String, String> metadata
     ) {
         this.fileName = fileName;
         this.streamContextSupplier = streamContextSupplier;
@@ -59,6 +62,7 @@ public WriteContext(
         this.uploadFinalizer = uploadFinalizer;
         this.doRemoteDataIntegrityCheck = doRemoteDataIntegrityCheck;
         this.expectedChecksum = expectedChecksum;
+        this.metadata = metadata;
     }
 
     /**
@@ -73,6 +77,7 @@ protected WriteContext(WriteContext writeContext) {
         this.uploadFinalizer = writeContext.uploadFinalizer;
         this.doRemoteDataIntegrityCheck = writeContext.doRemoteDataIntegrityCheck;
         this.expectedChecksum = writeContext.expectedChecksum;
+        this.metadata = writeContext.metadata;
     }
 
     /**
@@ -131,4 +136,87 @@ public boolean doRemoteDataIntegrityCheck() {
     public Long getExpectedChecksum() {
         return expectedChecksum;
     }
+
+    /**
+     * @return the upload metadata.
+     */
+    public Map<String, String> getMetadata() {
+        return metadata;
+    }
+
+    /**
+     * Builder for {@link WriteContext}.
+     *
+     * @opensearch.internal
+     */
+    public static class Builder {
+        private String fileName;
+        private StreamContextSupplier streamContextSupplier;
+        private long fileSize;
+        private boolean failIfAlreadyExists;
+        private WritePriority writePriority;
+        private CheckedConsumer<Boolean, IOException> uploadFinalizer;
+        private boolean doRemoteDataIntegrityCheck;
+        private Long expectedChecksum;
+        private Map<String, String> metadata;
+
+        public Builder fileName(String fileName) {
+            this.fileName = fileName;
+            return this;
+        }
+
+        public Builder streamContextSupplier(StreamContextSupplier streamContextSupplier) {
+            this.streamContextSupplier = streamContextSupplier;
+            return this;
+        }
+
+        public Builder fileSize(long fileSize) {
+            this.fileSize = fileSize;
+            return this;
+        }
+
+        public Builder writePriority(WritePriority writePriority) {
+            this.writePriority = writePriority;
+            return this;
+        }
+
+        public Builder failIfAlreadyExists(boolean failIfAlreadyExists) {
+            this.failIfAlreadyExists = failIfAlreadyExists;
+            return this;
+        }
+
+        public Builder uploadFinalizer(CheckedConsumer<Boolean, IOException> uploadFinalizer) {
+            this.uploadFinalizer = uploadFinalizer;
+            return this;
+        }
+
+        public Builder doRemoteDataIntegrityCheck(boolean doRemoteDataIntegrityCheck) {
+            this.doRemoteDataIntegrityCheck = doRemoteDataIntegrityCheck;
+            return this;
+        }
+
+        public Builder expectedChecksum(Long expectedChecksum) {
+            this.expectedChecksum = expectedChecksum;
+            return this;
+        }
+
+        public Builder metadata(Map<String, String> metadata) {
+            this.metadata = metadata;
+            return this;
+        }
+
+        public WriteContext build() {
+            return new WriteContext(
+                fileName,
+                streamContextSupplier,
+                fileSize,
+                failIfAlreadyExists,
+                writePriority,
+                uploadFinalizer,
+                doRemoteDataIntegrityCheck,
+                expectedChecksum,
+                metadata
+            );
+        }
+    }
 }
diff --git a/server/src/main/java/org/opensearch/common/blobstore/stream/write/WritePriority.java b/server/src/main/java/org/opensearch/common/blobstore/stream/write/WritePriority.java
index 3f341c878c3c7..4e8db0a3a8c69 100644
--- a/server/src/main/java/org/opensearch/common/blobstore/stream/write/WritePriority.java
+++ b/server/src/main/java/org/opensearch/common/blobstore/stream/write/WritePriority.java
@@ -14,7 +14,12 @@
  * @opensearch.internal
  */
 public enum WritePriority {
+    // Used for segment transfers during refresh, flush or merges
     NORMAL,
+    // Used for transfer of translog or ckp files.
     HIGH,
-    URGENT
+    // Used for transfer of remote cluster state
+    URGENT,
+    // All other background transfers such as in snapshot recovery, recovery from local store or index etc.
+    LOW
 }
diff --git a/server/src/main/java/org/opensearch/common/blobstore/transfer/RemoteTransferContainer.java b/server/src/main/java/org/opensearch/common/blobstore/transfer/RemoteTransferContainer.java
index 2047c99d9e13b..cbd1852202d1c 100644
--- a/server/src/main/java/org/opensearch/common/blobstore/transfer/RemoteTransferContainer.java
+++ b/server/src/main/java/org/opensearch/common/blobstore/transfer/RemoteTransferContainer.java
@@ -27,6 +27,7 @@
 import java.io.Closeable;
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.Map;
 import java.util.Objects;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Supplier;
@@ -51,10 +52,11 @@ public class RemoteTransferContainer implements Closeable {
     private final String remoteFileName;
     private final boolean failTransferIfFileExists;
     private final WritePriority writePriority;
-    private final long expectedChecksum;
+    private final Long expectedChecksum;
     private final OffsetRangeInputStreamSupplier offsetRangeInputStreamSupplier;
     private final boolean isRemoteDataIntegritySupported;
     private final AtomicBoolean readBlock = new AtomicBoolean();
+    private final Map<String, String> metadata;
 
     private static final Logger log = LogManager.getLogger(RemoteTransferContainer.class);
 
@@ -77,8 +79,45 @@ public RemoteTransferContainer(
         boolean failTransferIfFileExists,
         WritePriority writePriority,
         OffsetRangeInputStreamSupplier offsetRangeInputStreamSupplier,
-        long expectedChecksum,
+        Long expectedChecksum,
         boolean isRemoteDataIntegritySupported
+    ) {
+        this(
+            fileName,
+            remoteFileName,
+            contentLength,
+            failTransferIfFileExists,
+            writePriority,
+            offsetRangeInputStreamSupplier,
+            expectedChecksum,
+            isRemoteDataIntegritySupported,
+            null
+        );
+    }
+
+    /**
+     * Construct a new RemoteTransferContainer object with metadata.
+     *
+     * @param fileName                       Name of the local file
+     * @param remoteFileName                 Name of the remote file
+     * @param contentLength                  Total content length of the file to be uploaded
+     * @param failTransferIfFileExists       A boolean to determine if upload has to be failed if file exists
+     * @param writePriority                  The {@link WritePriority} of current upload
+     * @param offsetRangeInputStreamSupplier A supplier to create OffsetRangeInputStreams
+     * @param expectedChecksum               The expected checksum value for the file being uploaded. This checksum will be used for local or remote data integrity checks
+     * @param isRemoteDataIntegritySupported A boolean to signify whether the remote repository supports server side data integrity verification
+     * @param metadata                       Object metadata to be store with the file.
+     */
+    public RemoteTransferContainer(
+        String fileName,
+        String remoteFileName,
+        long contentLength,
+        boolean failTransferIfFileExists,
+        WritePriority writePriority,
+        OffsetRangeInputStreamSupplier offsetRangeInputStreamSupplier,
+        Long expectedChecksum,
+        boolean isRemoteDataIntegritySupported,
+        Map<String, String> metadata
     ) {
         this.fileName = fileName;
         this.remoteFileName = remoteFileName;
@@ -88,22 +127,23 @@ public RemoteTransferContainer(
         this.offsetRangeInputStreamSupplier = offsetRangeInputStreamSupplier;
         this.expectedChecksum = expectedChecksum;
         this.isRemoteDataIntegritySupported = isRemoteDataIntegritySupported;
+        this.metadata = metadata;
     }
 
     /**
      * @return The {@link  WriteContext} for the current upload
      */
     public WriteContext createWriteContext() {
-        return new WriteContext(
-            remoteFileName,
-            this::supplyStreamContext,
-            contentLength,
-            failTransferIfFileExists,
-            writePriority,
-            this::finalizeUpload,
-            isRemoteDataIntegrityCheckPossible(),
-            isRemoteDataIntegrityCheckPossible() ? expectedChecksum : null
-        );
+        return new WriteContext.Builder().fileName(remoteFileName)
+            .streamContextSupplier(this::supplyStreamContext)
+            .fileSize(contentLength)
+            .failIfAlreadyExists(failTransferIfFileExists)
+            .writePriority(writePriority)
+            .uploadFinalizer(this::finalizeUpload)
+            .doRemoteDataIntegrityCheck(isRemoteDataIntegrityCheckPossible())
+            .expectedChecksum(isRemoteDataIntegrityCheckPossible() ? expectedChecksum : null)
+            .metadata(metadata)
+            .build();
     }
 
     // package-private for testing
@@ -190,7 +230,7 @@ private LocalStreamSupplier<InputStreamContainer> getMultipartStreamSupplier(
     }
 
     private boolean isRemoteDataIntegrityCheckPossible() {
-        return isRemoteDataIntegritySupported;
+        return isRemoteDataIntegritySupported && Objects.nonNull(expectedChecksum);
     }
 
     private void finalizeUpload(boolean uploadSuccessful) throws IOException {
@@ -198,7 +238,7 @@ private void finalizeUpload(boolean uploadSuccessful) throws IOException {
             return;
         }
 
-        if (uploadSuccessful) {
+        if (uploadSuccessful && Objects.nonNull(expectedChecksum)) {
             long actualChecksum = getActualChecksum();
             if (actualChecksum != expectedChecksum) {
                 throw new CorruptIndexException(
diff --git a/server/src/main/java/org/opensearch/common/cache/Cache.java b/server/src/main/java/org/opensearch/common/cache/Cache.java
index d8aa4e93735e6..6d346de25cadf 100644
--- a/server/src/main/java/org/opensearch/common/cache/Cache.java
+++ b/server/src/main/java/org/opensearch/common/cache/Cache.java
@@ -896,6 +896,10 @@ private void relinkAtHead(Entry<K, V> entry) {
         }
     }
 
+    public ToLongBiFunction<K, V> getWeigher() {
+        return weigher;
+    }
+
     private CacheSegment<K, V> getCacheSegment(K key) {
         return segments[key.hashCode() & 0xff];
     }
diff --git a/server/src/main/java/org/opensearch/common/cache/CacheType.java b/server/src/main/java/org/opensearch/common/cache/CacheType.java
index c5aeb7cd1fa40..eee6204ac5412 100644
--- a/server/src/main/java/org/opensearch/common/cache/CacheType.java
+++ b/server/src/main/java/org/opensearch/common/cache/CacheType.java
@@ -10,20 +10,52 @@
 
 import org.opensearch.common.annotation.ExperimentalApi;
 
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
 /**
  * Cache types available within OpenSearch.
  */
 @ExperimentalApi
 public enum CacheType {
-    INDICES_REQUEST_CACHE("indices.requests.cache");
+    INDICES_REQUEST_CACHE("indices.requests.cache", "request_cache");
 
     private final String settingPrefix;
+    private final String value; // The value displayed for this cache type in stats API responses
+
+    private static final Map<String, CacheType> valuesMap;
+    static {
+        Map<String, CacheType> values = new HashMap<>();
+        for (CacheType cacheType : values()) {
+            values.put(cacheType.value, cacheType);
+        }
+        valuesMap = Collections.unmodifiableMap(values);
+    }
 
-    CacheType(String settingPrefix) {
+    CacheType(String settingPrefix, String representation) {
         this.settingPrefix = settingPrefix;
+        this.value = representation;
     }
 
     public String getSettingPrefix() {
         return settingPrefix;
     }
+
+    public String getValue() {
+        return value;
+    }
+
+    public static CacheType getByValue(String value) {
+        CacheType result = valuesMap.get(value);
+        if (result == null) {
+            throw new IllegalArgumentException("No CacheType with value = " + value);
+        }
+        return result;
+    }
+
+    public static Set<String> allValues() {
+        return valuesMap.keySet();
+    }
 }
diff --git a/server/src/main/java/org/opensearch/common/cache/ICache.java b/server/src/main/java/org/opensearch/common/cache/ICache.java
index f7be46a852631..f5dd644db6d6b 100644
--- a/server/src/main/java/org/opensearch/common/cache/ICache.java
+++ b/server/src/main/java/org/opensearch/common/cache/ICache.java
@@ -9,6 +9,7 @@
 package org.opensearch.common.cache;
 
 import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.common.cache.stats.ImmutableCacheStatsHolder;
 import org.opensearch.common.cache.store.config.CacheConfig;
 
 import java.io.Closeable;
@@ -23,22 +24,35 @@
  */
 @ExperimentalApi
 public interface ICache<K, V> extends Closeable {
-    V get(K key);
+    V get(ICacheKey<K> key);
 
-    void put(K key, V value);
+    void put(ICacheKey<K> key, V value);
 
-    V computeIfAbsent(K key, LoadAwareCacheLoader<K, V> loader) throws Exception;
+    V computeIfAbsent(ICacheKey<K> key, LoadAwareCacheLoader<ICacheKey<K>, V> loader) throws Exception;
 
-    void invalidate(K key);
+    /**
+     * Invalidates the key. If a dimension in the key has dropStatsOnInvalidation set to true, the cache also completely
+     * resets stats for that dimension value. It's the caller's responsibility to make sure all keys with that dimension value are
+     * actually invalidated.
+     */
+    void invalidate(ICacheKey<K> key);
 
     void invalidateAll();
 
-    Iterable<K> keys();
+    Iterable<ICacheKey<K>> keys();
 
     long count();
 
     void refresh();
 
+    // Return total stats only
+    default ImmutableCacheStatsHolder stats() {
+        return stats(null);
+    }
+
+    // Return stats aggregated by the provided levels. If levels is null or an empty array, return total stats only.
+    ImmutableCacheStatsHolder stats(String[] levels);
+
     /**
      * Factory to create objects.
      */
diff --git a/server/src/main/java/org/opensearch/common/cache/ICacheKey.java b/server/src/main/java/org/opensearch/common/cache/ICacheKey.java
new file mode 100644
index 0000000000000..4d93aab933751
--- /dev/null
+++ b/server/src/main/java/org/opensearch/common/cache/ICacheKey.java
@@ -0,0 +1,96 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.cache;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+
+import java.util.List;
+
+/**
+ * A key wrapper used for ICache implementations, which carries dimensions with it.
+ * @param <K> the type of the underlying key. K must implement equals(), or else ICacheKey.equals()
+ *           won't work properly and cache behavior may be incorrect!
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public class ICacheKey<K> {
+    public final K key; // K must implement equals()
+    public final List<String> dimensions; // Dimension values. The dimension names are implied.
+    /**
+     * If this key is invalidated and dropDimensions is true, the ICache implementation will also drop all stats,
+     * including hits/misses/evictions, with this combination of dimension values.
+     */
+    private boolean dropStatsForDimensions;
+
+    /**
+     * Constructor to use when specifying dimensions.
+     */
+    public ICacheKey(K key, List<String> dimensions) {
+        this.key = key;
+        this.dimensions = dimensions;
+    }
+
+    /**
+     * Constructor to use when no dimensions are needed.
+     */
+    public ICacheKey(K key) {
+        this.key = key;
+        this.dimensions = List.of();
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (o == this) {
+            return true;
+        }
+        if (o == null) {
+            return false;
+        }
+        if (o.getClass() != ICacheKey.class) {
+            return false;
+        }
+        ICacheKey other = (ICacheKey) o;
+        if (!dimensions.equals(other.dimensions)) {
+            return false;
+        }
+        if (this.key == null && other.key == null) {
+            return true;
+        }
+        if (this.key == null || other.key == null) {
+            return false;
+        }
+        return this.key.equals(other.key);
+    }
+
+    @Override
+    public int hashCode() {
+        if (key == null) {
+            return dimensions.hashCode();
+        }
+        return 31 * key.hashCode() + dimensions.hashCode();
+    }
+
+    // As K might not be Accountable, directly pass in its memory usage to be added.
+    public long ramBytesUsed(long underlyingKeyRamBytes) {
+        long estimate = underlyingKeyRamBytes;
+        for (String dim : dimensions) {
+            estimate += dim.length();
+        }
+        return estimate;
+    }
+
+    public boolean getDropStatsForDimensions() {
+        return dropStatsForDimensions;
+    }
+
+    public void setDropStatsForDimensions(boolean newValue) {
+        dropStatsForDimensions = newValue;
+    }
+}
diff --git a/server/src/main/java/org/opensearch/common/cache/RemovalListener.java b/server/src/main/java/org/opensearch/common/cache/RemovalListener.java
index 369313f9f93f4..68e1cdf6139e2 100644
--- a/server/src/main/java/org/opensearch/common/cache/RemovalListener.java
+++ b/server/src/main/java/org/opensearch/common/cache/RemovalListener.java
@@ -32,11 +32,14 @@
 
 package org.opensearch.common.cache;
 
+import org.opensearch.common.annotation.ExperimentalApi;
+
 /**
  * Listener for removing an element from the cache
  *
- * @opensearch.internal
+ * @opensearch.experimental
  */
+@ExperimentalApi
 @FunctionalInterface
 public interface RemovalListener<K, V> {
     void onRemoval(RemovalNotification<K, V> notification);
diff --git a/server/src/main/java/org/opensearch/common/cache/policy/CachedQueryResult.java b/server/src/main/java/org/opensearch/common/cache/policy/CachedQueryResult.java
index 0a98542a05bb7..df698112c60d1 100644
--- a/server/src/main/java/org/opensearch/common/cache/policy/CachedQueryResult.java
+++ b/server/src/main/java/org/opensearch/common/cache/policy/CachedQueryResult.java
@@ -8,6 +8,7 @@
 
 package org.opensearch.common.cache.policy;
 
+import org.opensearch.common.annotation.ExperimentalApi;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput;
 import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
@@ -62,7 +63,10 @@ public void writeToNoId(StreamOutput out) throws IOException {
     /**
      * A class containing information needed for all cache policies
      *  to decide whether to admit a given value.
+     *
+     *  @opensearch.experimental
      */
+    @ExperimentalApi
     public static class PolicyValues implements Writeable {
         final long tookTimeNanos;
         // More values can be added here as they're needed for future policies
diff --git a/server/src/main/java/org/opensearch/common/cache/serializer/ICacheKeySerializer.java b/server/src/main/java/org/opensearch/common/cache/serializer/ICacheKeySerializer.java
new file mode 100644
index 0000000000000..7521e23091464
--- /dev/null
+++ b/server/src/main/java/org/opensearch/common/cache/serializer/ICacheKeySerializer.java
@@ -0,0 +1,87 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.cache.serializer;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.opensearch.OpenSearchException;
+import org.opensearch.common.cache.ICacheKey;
+import org.opensearch.common.io.stream.BytesStreamOutput;
+import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.core.common.io.stream.BytesStreamInput;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * A serializer for ICacheKey.
+ * @param <K> the type of the underlying key in ICacheKey
+ */
+public class ICacheKeySerializer<K> implements Serializer<ICacheKey<K>, byte[]> {
+
+    public final Serializer<K, byte[]> keySerializer;
+    private final Logger logger = LogManager.getLogger(ICacheKeySerializer.class);
+
+    public ICacheKeySerializer(Serializer<K, byte[]> serializer) {
+        this.keySerializer = serializer;
+    }
+
+    @Override
+    public byte[] serialize(ICacheKey<K> object) {
+        if (object == null || object.key == null || object.dimensions == null) {
+            return null;
+        }
+        byte[] serializedKey = keySerializer.serialize(object.key);
+        try {
+            BytesStreamOutput os = new BytesStreamOutput();
+            // First write the number of dimensions
+            os.writeVInt(object.dimensions.size());
+            for (String dimValue : object.dimensions) {
+                os.writeString(dimValue);
+            }
+            os.writeVInt(serializedKey.length); // The read byte[] fn seems to not work as expected
+            os.writeBytes(serializedKey);
+            byte[] finalBytes = BytesReference.toBytes(os.bytes());
+            return finalBytes;
+        } catch (IOException e) {
+            logger.debug("Could not write ICacheKey to byte[]");
+            throw new OpenSearchException(e);
+        }
+    }
+
+    @Override
+    public ICacheKey<K> deserialize(byte[] bytes) {
+        if (bytes == null) {
+            return null;
+        }
+        List<String> dimensionList = new ArrayList<>();
+        try {
+            BytesStreamInput is = new BytesStreamInput(bytes, 0, bytes.length);
+            int numDimensions = is.readVInt();
+            for (int i = 0; i < numDimensions; i++) {
+                dimensionList.add(is.readString());
+            }
+
+            int length = is.readVInt();
+            byte[] serializedKey = new byte[length];
+            is.readBytes(serializedKey, 0, length);
+            return new ICacheKey<>(keySerializer.deserialize(serializedKey), dimensionList);
+        } catch (IOException e) {
+            logger.debug("Could not write byte[] to ICacheKey");
+            throw new OpenSearchException(e);
+        }
+    }
+
+    @Override
+    public boolean equals(ICacheKey<K> object, byte[] bytes) {
+        return Arrays.equals(serialize(object), bytes);
+    }
+}
diff --git a/server/src/main/java/org/opensearch/common/cache/serializer/Serializer.java b/server/src/main/java/org/opensearch/common/cache/serializer/Serializer.java
index 35e28707d1ca3..46a8ed5a72ccf 100644
--- a/server/src/main/java/org/opensearch/common/cache/serializer/Serializer.java
+++ b/server/src/main/java/org/opensearch/common/cache/serializer/Serializer.java
@@ -8,10 +8,15 @@
 
 package org.opensearch.common.cache.serializer;
 
+import org.opensearch.common.annotation.ExperimentalApi;
+
 /**
  * Defines an interface for serializers, to be used by pluggable caches.
  * T is the class of the original object, and U is the serialized class.
+ *
+ * @opensearch.experimental
  */
+@ExperimentalApi
 public interface Serializer<T, U> {
     /**
      * Serializes an object.
diff --git a/server/src/main/java/org/opensearch/common/cache/service/CacheService.java b/server/src/main/java/org/opensearch/common/cache/service/CacheService.java
index b6710e5e4b424..01da78ecec52e 100644
--- a/server/src/main/java/org/opensearch/common/cache/service/CacheService.java
+++ b/server/src/main/java/org/opensearch/common/cache/service/CacheService.java
@@ -8,10 +8,12 @@
 
 package org.opensearch.common.cache.service;
 
+import org.opensearch.action.admin.indices.stats.CommonStatsFlags;
 import org.opensearch.common.annotation.ExperimentalApi;
 import org.opensearch.common.cache.CacheType;
 import org.opensearch.common.cache.ICache;
 import org.opensearch.common.cache.settings.CacheSettings;
+import org.opensearch.common.cache.stats.ImmutableCacheStatsHolder;
 import org.opensearch.common.cache.store.OpenSearchOnHeapCache;
 import org.opensearch.common.cache.store.config.CacheConfig;
 import org.opensearch.common.settings.Setting;
@@ -20,6 +22,8 @@
 
 import java.util.HashMap;
 import java.util.Map;
+import java.util.SortedMap;
+import java.util.TreeMap;
 
 /**
  * Service responsible to create caches.
@@ -62,4 +66,12 @@ public <K, V> ICache<K, V> createCache(CacheConfig<K, V> config, CacheType cache
         cacheTypeMap.put(cacheType, iCache);
         return iCache;
     }
+
+    public NodeCacheStats stats(CommonStatsFlags flags) {
+        final SortedMap<CacheType, ImmutableCacheStatsHolder> statsMap = new TreeMap<>();
+        for (CacheType type : cacheTypeMap.keySet()) {
+            statsMap.put(type, cacheTypeMap.get(type).stats(flags.getLevels()));
+        }
+        return new NodeCacheStats(statsMap, flags);
+    }
 }
diff --git a/server/src/main/java/org/opensearch/common/cache/service/NodeCacheStats.java b/server/src/main/java/org/opensearch/common/cache/service/NodeCacheStats.java
new file mode 100644
index 0000000000000..07c75eab34194
--- /dev/null
+++ b/server/src/main/java/org/opensearch/common/cache/service/NodeCacheStats.java
@@ -0,0 +1,80 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.cache.service;
+
+import org.opensearch.action.admin.indices.stats.CommonStatsFlags;
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.common.cache.CacheType;
+import org.opensearch.common.cache.stats.ImmutableCacheStatsHolder;
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.core.common.io.stream.Writeable;
+import org.opensearch.core.xcontent.ToXContentFragment;
+import org.opensearch.core.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Objects;
+import java.util.SortedMap;
+
+/**
+ * A class creating XContent responses to cache stats API requests.
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public class NodeCacheStats implements ToXContentFragment, Writeable {
+    // Use SortedMap to force consistent ordering of caches in API responses
+    private final SortedMap<CacheType, ImmutableCacheStatsHolder> statsByCache;
+    private final CommonStatsFlags flags;
+
+    public NodeCacheStats(SortedMap<CacheType, ImmutableCacheStatsHolder> statsByCache, CommonStatsFlags flags) {
+        this.statsByCache = statsByCache;
+        this.flags = flags;
+    }
+
+    public NodeCacheStats(StreamInput in) throws IOException {
+        this.flags = new CommonStatsFlags(in);
+        this.statsByCache = in.readOrderedMap(i -> i.readEnum(CacheType.class), ImmutableCacheStatsHolder::new);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        flags.writeTo(out);
+        out.writeMap(statsByCache, StreamOutput::writeEnum, (o, immutableCacheStatsHolder) -> immutableCacheStatsHolder.writeTo(o));
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        for (CacheType type : statsByCache.keySet()) {
+            if (flags.getIncludeCaches().contains(type)) {
+                builder.startObject(type.getValue());
+                statsByCache.get(type).toXContent(builder, params);
+                builder.endObject();
+            }
+        }
+        return builder;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (o == null) {
+            return false;
+        }
+        if (o.getClass() != NodeCacheStats.class) {
+            return false;
+        }
+        NodeCacheStats other = (NodeCacheStats) o;
+        return statsByCache.equals(other.statsByCache) && flags.getIncludeCaches().equals(other.flags.getIncludeCaches());
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(statsByCache, flags);
+    }
+}
diff --git a/server/src/main/java/org/opensearch/common/cache/stats/CacheStats.java b/server/src/main/java/org/opensearch/common/cache/stats/CacheStats.java
new file mode 100644
index 0000000000000..93fa1ff7fcddf
--- /dev/null
+++ b/server/src/main/java/org/opensearch/common/cache/stats/CacheStats.java
@@ -0,0 +1,132 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.cache.stats;
+
+import org.opensearch.common.metrics.CounterMetric;
+
+import java.util.Objects;
+
+/**
+ * A mutable class containing the 5 live metrics tracked by a StatsHolder object.
+ */
+public class CacheStats {
+    CounterMetric hits;
+    CounterMetric misses;
+    CounterMetric evictions;
+    CounterMetric sizeInBytes;
+    CounterMetric items;
+
+    public CacheStats(long hits, long misses, long evictions, long sizeInBytes, long items) {
+        this.hits = new CounterMetric();
+        this.hits.inc(hits);
+        this.misses = new CounterMetric();
+        this.misses.inc(misses);
+        this.evictions = new CounterMetric();
+        this.evictions.inc(evictions);
+        this.sizeInBytes = new CounterMetric();
+        this.sizeInBytes.inc(sizeInBytes);
+        this.items = new CounterMetric();
+        this.items.inc(items);
+    }
+
+    public CacheStats() {
+        this(0, 0, 0, 0, 0);
+    }
+
+    private void internalAdd(long otherHits, long otherMisses, long otherEvictions, long otherSizeInBytes, long otherEntries) {
+        this.hits.inc(otherHits);
+        this.misses.inc(otherMisses);
+        this.evictions.inc(otherEvictions);
+        this.sizeInBytes.inc(otherSizeInBytes);
+        this.items.inc(otherEntries);
+    }
+
+    public void add(CacheStats other) {
+        if (other == null) {
+            return;
+        }
+        internalAdd(other.getHits(), other.getMisses(), other.getEvictions(), other.getSizeInBytes(), other.getItems());
+    }
+
+    public void add(ImmutableCacheStats snapshot) {
+        if (snapshot == null) {
+            return;
+        }
+        internalAdd(snapshot.getHits(), snapshot.getMisses(), snapshot.getEvictions(), snapshot.getSizeInBytes(), snapshot.getItems());
+    }
+
+    public void subtract(ImmutableCacheStats other) {
+        if (other == null) {
+            return;
+        }
+        internalAdd(-other.getHits(), -other.getMisses(), -other.getEvictions(), -other.getSizeInBytes(), -other.getItems());
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(hits.count(), misses.count(), evictions.count(), sizeInBytes.count(), items.count());
+    }
+
+    public void incrementHits() {
+        hits.inc();
+    }
+
+    public void incrementMisses() {
+        misses.inc();
+    }
+
+    public void incrementEvictions() {
+        evictions.inc();
+    }
+
+    public void incrementSizeInBytes(long amount) {
+        sizeInBytes.inc(amount);
+    }
+
+    public void decrementSizeInBytes(long amount) {
+        sizeInBytes.dec(amount);
+    }
+
+    public void incrementItems() {
+        items.inc();
+    }
+
+    public void decrementItems() {
+        items.dec();
+    }
+
+    public long getHits() {
+        return hits.count();
+    }
+
+    public long getMisses() {
+        return misses.count();
+    }
+
+    public long getEvictions() {
+        return evictions.count();
+    }
+
+    public long getSizeInBytes() {
+        return sizeInBytes.count();
+    }
+
+    public long getItems() {
+        return items.count();
+    }
+
+    public void resetSizeAndEntries() {
+        sizeInBytes = new CounterMetric();
+        items = new CounterMetric();
+    }
+
+    public ImmutableCacheStats immutableSnapshot() {
+        return new ImmutableCacheStats(hits.count(), misses.count(), evictions.count(), sizeInBytes.count(), items.count());
+    }
+}
diff --git a/server/src/main/java/org/opensearch/common/cache/stats/CacheStatsHolder.java b/server/src/main/java/org/opensearch/common/cache/stats/CacheStatsHolder.java
new file mode 100644
index 0000000000000..27cb7679efb0c
--- /dev/null
+++ b/server/src/main/java/org/opensearch/common/cache/stats/CacheStatsHolder.java
@@ -0,0 +1,38 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.cache.stats;
+
+import java.util.List;
+
+/**
+ * An interface extended by DefaultCacheStatsHolder and NoopCacheStatsHolder.
+ */
+public interface CacheStatsHolder {
+    void incrementHits(List<String> dimensionValues);
+
+    void incrementMisses(List<String> dimensionValues);
+
+    void incrementEvictions(List<String> dimensionValues);
+
+    void incrementSizeInBytes(List<String> dimensionValues, long amountBytes);
+
+    void decrementSizeInBytes(List<String> dimensionValues, long amountBytes);
+
+    void incrementItems(List<String> dimensionValues);
+
+    void decrementItems(List<String> dimensionValues);
+
+    void reset();
+
+    long count();
+
+    void removeDimensions(List<String> dimensionValues);
+
+    ImmutableCacheStatsHolder getImmutableCacheStatsHolder(String[] levels);
+}
diff --git a/server/src/main/java/org/opensearch/common/cache/stats/DefaultCacheStatsHolder.java b/server/src/main/java/org/opensearch/common/cache/stats/DefaultCacheStatsHolder.java
new file mode 100644
index 0000000000000..ea92c8e81b8f0
--- /dev/null
+++ b/server/src/main/java/org/opensearch/common/cache/stats/DefaultCacheStatsHolder.java
@@ -0,0 +1,314 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.cache.stats;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+import java.util.function.Consumer;
+
+/**
+ * A class ICache implementations use to internally keep track of their stats across multiple dimensions.
+ * Not intended to be exposed outside the cache; for this, caches use getImmutableCacheStatsHolder() to create an immutable
+ * copy of the current state of the stats.
+ * Currently, in the IRC, the stats tracked in a CacheStatsHolder will not appear for empty shards that have had no cache
+ * operations done on them yet. This might be changed in the future, by exposing a method to add empty nodes to the
+ * tree in CacheStatsHolder in the ICache interface.
+ *
+ * @opensearch.experimental
+ */
+public class DefaultCacheStatsHolder implements CacheStatsHolder {
+
+    // The list of permitted dimensions. Should be ordered from "outermost" to "innermost", as you would like to
+    // aggregate them in an API response.
+    protected final List<String> dimensionNames;
+    // A tree structure based on dimension values, which stores stats values in its leaf nodes.
+    // Non-leaf nodes have stats matching the sum of their children.
+    // We use a tree structure, rather than a map with concatenated keys, to save on memory usage. If there are many leaf
+    // nodes that share a parent, that parent's dimension value will only be stored once, not many times.
+    private final Node statsRoot;
+    // To avoid sync problems, obtain a lock before creating or removing nodes in the stats tree.
+    // No lock is needed to edit stats on existing nodes.
+    private final Lock lock = new ReentrantLock();
+    // The name of the cache type using these stats
+    private final String storeName;
+
+    public DefaultCacheStatsHolder(List<String> dimensionNames, String storeName) {
+        this.dimensionNames = Collections.unmodifiableList(dimensionNames);
+        this.storeName = storeName;
+        this.statsRoot = new Node("", true); // The root node has the empty string as its dimension value
+    }
+
+    public List<String> getDimensionNames() {
+        return dimensionNames;
+    }
+
+    // For all these increment functions, the dimensions list comes from the key, and contains all dimensions present in dimensionNames.
+    // The order has to match the order given in dimensionNames.
+    @Override
+    public void incrementHits(List<String> dimensionValues) {
+        internalIncrement(dimensionValues, Node::incrementHits, true);
+    }
+
+    @Override
+    public void incrementMisses(List<String> dimensionValues) {
+        internalIncrement(dimensionValues, Node::incrementMisses, true);
+    }
+
+    @Override
+    public void incrementEvictions(List<String> dimensionValues) {
+        internalIncrement(dimensionValues, Node::incrementEvictions, true);
+    }
+
+    @Override
+    public void incrementSizeInBytes(List<String> dimensionValues, long amountBytes) {
+        internalIncrement(dimensionValues, (node) -> node.incrementSizeInBytes(amountBytes), true);
+    }
+
+    // For decrements, we should not create nodes if they are absent. This protects us from erroneously decrementing values for keys
+    // which have been entirely deleted, for example in an async removal listener.
+    @Override
+    public void decrementSizeInBytes(List<String> dimensionValues, long amountBytes) {
+        internalIncrement(dimensionValues, (node) -> node.decrementSizeInBytes(amountBytes), false);
+    }
+
+    @Override
+    public void incrementItems(List<String> dimensionValues) {
+        internalIncrement(dimensionValues, Node::incrementItems, true);
+    }
+
+    @Override
+    public void decrementItems(List<String> dimensionValues) {
+        internalIncrement(dimensionValues, Node::decrementItems, false);
+    }
+
+    /**
+     * Reset number of entries and memory size when all keys leave the cache, but don't reset hit/miss/eviction numbers.
+     * This is in line with the behavior of the existing API when caches are cleared.
+     */
+    @Override
+    public void reset() {
+        resetHelper(statsRoot);
+    }
+
+    private void resetHelper(Node current) {
+        current.resetSizeAndEntries();
+        for (Node child : current.children.values()) {
+            resetHelper(child);
+        }
+    }
+
+    @Override
+    public long count() {
+        // Include this here so caches don't have to create an entire CacheStats object to run count().
+        return statsRoot.getEntries();
+    }
+
+    protected void internalIncrement(List<String> dimensionValues, Consumer<Node> adder, boolean createNodesIfAbsent) {
+        assert dimensionValues.size() == dimensionNames.size();
+        // First try to increment without creating nodes
+        boolean didIncrement = internalIncrementHelper(dimensionValues, statsRoot, 0, adder, false);
+        // If we failed to increment, because nodes had to be created, obtain the lock and run again while creating nodes if needed
+        if (!didIncrement && createNodesIfAbsent) {
+            try {
+                lock.lock();
+                internalIncrementHelper(dimensionValues, statsRoot, 0, adder, true);
+            } finally {
+                lock.unlock();
+            }
+        }
+    }
+
+    /**
+     * Use the incrementer function to increment/decrement a value in the stats for a set of dimensions.
+     * If createNodesIfAbsent is true, and there is no stats for this set of dimensions, create one.
+     * Returns true if the increment was applied, false if not.
+     */
+    private boolean internalIncrementHelper(
+        List<String> dimensionValues,
+        Node node,
+        int depth, // Pass in the depth to avoid having to slice the list for each node.
+        Consumer<Node> adder,
+        boolean createNodesIfAbsent
+    ) {
+        if (depth == dimensionValues.size()) {
+            // This is the leaf node we are trying to reach
+            adder.accept(node);
+            return true;
+        }
+
+        Node child = node.getChild(dimensionValues.get(depth));
+        if (child == null) {
+            if (createNodesIfAbsent) {
+                boolean createMapInChild = depth < dimensionValues.size() - 1;
+                child = node.createChild(dimensionValues.get(depth), createMapInChild);
+            } else {
+                return false;
+            }
+        }
+        if (internalIncrementHelper(dimensionValues, child, depth + 1, adder, createNodesIfAbsent)) {
+            // Function returns true if the next node down was incremented
+            adder.accept(node);
+            return true;
+        }
+        return false;
+    }
+
+    /**
+     * Produce an immutable version of these stats, aggregated according to levels.
+     * If levels is null, do not aggregate and return an immutable version of the original tree.
+     */
+    @Override
+    public ImmutableCacheStatsHolder getImmutableCacheStatsHolder(String[] levels) {
+        String[] nonNullLevels = Objects.requireNonNullElseGet(levels, () -> new String[0]);
+        return new ImmutableCacheStatsHolder(this.statsRoot, nonNullLevels, dimensionNames, storeName);
+    }
+
+    @Override
+    public void removeDimensions(List<String> dimensionValues) {
+        assert dimensionValues.size() == dimensionNames.size() : "Must specify a value for every dimension when removing from StatsHolder";
+        // As we are removing nodes from the tree, obtain the lock
+        lock.lock();
+        try {
+            removeDimensionsHelper(dimensionValues, statsRoot, 0);
+        } finally {
+            lock.unlock();
+        }
+    }
+
+    // Returns a CacheStatsCounterSnapshot object for the stats to decrement if the removal happened, null otherwise.
+    private ImmutableCacheStats removeDimensionsHelper(List<String> dimensionValues, Node node, int depth) {
+        if (depth == dimensionValues.size()) {
+            // Pass up a snapshot of the original stats to avoid issues when the original is decremented by other fn invocations
+            return node.getImmutableStats();
+        }
+        Node child = node.getChild(dimensionValues.get(depth));
+        if (child == null) {
+            return null;
+        }
+        ImmutableCacheStats statsToDecrement = removeDimensionsHelper(dimensionValues, child, depth + 1);
+        if (statsToDecrement != null) {
+            // The removal took place, decrement values and remove this node from its parent if it's now empty
+            node.decrementBySnapshot(statsToDecrement);
+            if (child.getChildren().isEmpty()) {
+                node.children.remove(child.getDimensionValue());
+            }
+        }
+        return statsToDecrement;
+    }
+
+    // pkg-private for testing
+    Node getStatsRoot() {
+        return statsRoot;
+    }
+
+    /**
+     * Nodes that make up the tree in the stats holder.
+     */
+    protected static class Node {
+        private final String dimensionValue;
+        // Map from dimensionValue to the DimensionNode for that dimension value.
+        final Map<String, Node> children;
+        // The stats for this node. If a leaf node, corresponds to the stats for this combination of dimensions; if not,
+        // contains the sum of its children's stats.
+        private CacheStats stats;
+
+        // Used for leaf nodes to avoid allocating many unnecessary maps
+        private static final Map<String, Node> EMPTY_CHILDREN_MAP = new HashMap<>();
+
+        Node(String dimensionValue, boolean createChildrenMap) {
+            this.dimensionValue = dimensionValue;
+            if (createChildrenMap) {
+                this.children = new ConcurrentHashMap<>();
+            } else {
+                this.children = EMPTY_CHILDREN_MAP;
+            }
+            this.stats = new CacheStats();
+        }
+
+        public String getDimensionValue() {
+            return dimensionValue;
+        }
+
+        protected Map<String, Node> getChildren() {
+            // We can safely iterate over ConcurrentHashMap without worrying about thread issues.
+            return children;
+        }
+
+        // Functions for modifying internal CacheStatsCounter without callers having to be aware of CacheStatsCounter
+
+        public void incrementHits() {
+            this.stats.incrementHits();
+        }
+
+        public void incrementMisses() {
+            this.stats.incrementMisses();
+        }
+
+        public void incrementEvictions() {
+            this.stats.incrementEvictions();
+        }
+
+        public void incrementSizeInBytes(long amountBytes) {
+            this.stats.incrementSizeInBytes(amountBytes);
+        }
+
+        public void decrementSizeInBytes(long amountBytes) {
+            this.stats.decrementSizeInBytes(amountBytes);
+        }
+
+        void incrementItems() {
+            this.stats.incrementItems();
+        }
+
+        void decrementItems() {
+            this.stats.decrementItems();
+        }
+
+        long getEntries() {
+            return this.stats.getItems();
+        }
+
+        ImmutableCacheStats getImmutableStats() {
+            return this.stats.immutableSnapshot();
+        }
+
+        void decrementBySnapshot(ImmutableCacheStats snapshot) {
+            this.stats.subtract(snapshot);
+        }
+
+        void resetSizeAndEntries() {
+            this.stats.resetSizeAndEntries();
+        }
+
+        Node getChild(String dimensionValue) {
+            return children.get(dimensionValue);
+        }
+
+        Node createChild(String dimensionValue, boolean createMapInChild) {
+            return children.computeIfAbsent(dimensionValue, (key) -> new Node(dimensionValue, createMapInChild));
+        }
+
+        /**
+         * Return whether this is a leaf node which is at the lowest level of the tree.
+         * Does not return true if this is a node at a higher level whose children are still being constructed.
+         * @return if this is a leaf node at the lowest level
+         */
+        public boolean isAtLowestLevel() {
+            // Compare by value to the empty children map, to ensure we don't get false positives for nodes
+            // which are in the process of having children added
+            return children == EMPTY_CHILDREN_MAP;
+        }
+    }
+}
diff --git a/server/src/main/java/org/opensearch/common/cache/stats/ImmutableCacheStats.java b/server/src/main/java/org/opensearch/common/cache/stats/ImmutableCacheStats.java
new file mode 100644
index 0000000000000..dbd78a2584f9c
--- /dev/null
+++ b/server/src/main/java/org/opensearch/common/cache/stats/ImmutableCacheStats.java
@@ -0,0 +1,129 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.cache.stats;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.core.common.io.stream.Writeable;
+import org.opensearch.core.common.unit.ByteSizeValue;
+import org.opensearch.core.xcontent.ToXContent;
+import org.opensearch.core.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * An immutable snapshot of CacheStats.
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public class ImmutableCacheStats implements Writeable, ToXContent {
+    private final long hits;
+    private final long misses;
+    private final long evictions;
+    private final long sizeInBytes;
+    private final long items;
+
+    public ImmutableCacheStats(long hits, long misses, long evictions, long sizeInBytes, long items) {
+        this.hits = hits;
+        this.misses = misses;
+        this.evictions = evictions;
+        this.sizeInBytes = sizeInBytes;
+        this.items = items;
+    }
+
+    public ImmutableCacheStats(StreamInput in) throws IOException {
+        this(in.readVLong(), in.readVLong(), in.readVLong(), in.readVLong(), in.readVLong());
+    }
+
+    public static ImmutableCacheStats addSnapshots(ImmutableCacheStats s1, ImmutableCacheStats s2) {
+        return new ImmutableCacheStats(
+            s1.hits + s2.hits,
+            s1.misses + s2.misses,
+            s1.evictions + s2.evictions,
+            s1.sizeInBytes + s2.sizeInBytes,
+            s1.items + s2.items
+        );
+    }
+
+    public long getHits() {
+        return hits;
+    }
+
+    public long getMisses() {
+        return misses;
+    }
+
+    public long getEvictions() {
+        return evictions;
+    }
+
+    public long getSizeInBytes() {
+        return sizeInBytes;
+    }
+
+    public long getItems() {
+        return items;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeVLong(hits);
+        out.writeVLong(misses);
+        out.writeVLong(evictions);
+        out.writeVLong(sizeInBytes);
+        out.writeVLong(items);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (o == null) {
+            return false;
+        }
+        if (o.getClass() != ImmutableCacheStats.class) {
+            return false;
+        }
+        ImmutableCacheStats other = (ImmutableCacheStats) o;
+        return (hits == other.hits)
+            && (misses == other.misses)
+            && (evictions == other.evictions)
+            && (sizeInBytes == other.sizeInBytes)
+            && (items == other.items);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(hits, misses, evictions, sizeInBytes, items);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        // We don't write the header in CacheStatsResponse's toXContent, because it doesn't know the name of aggregation it's part of
+        builder.humanReadableField(Fields.SIZE_IN_BYTES, Fields.SIZE, new ByteSizeValue(sizeInBytes));
+        builder.field(Fields.EVICTIONS, evictions);
+        builder.field(Fields.HIT_COUNT, hits);
+        builder.field(Fields.MISS_COUNT, misses);
+        builder.field(Fields.ITEM_COUNT, items);
+        return builder;
+    }
+
+    /**
+     * Field names used to write the values in this object to XContent.
+     */
+    public static final class Fields {
+        public static final String SIZE = "size";
+        public static final String SIZE_IN_BYTES = "size_in_bytes";
+        public static final String EVICTIONS = "evictions";
+        public static final String HIT_COUNT = "hit_count";
+        public static final String MISS_COUNT = "miss_count";
+        public static final String ITEM_COUNT = "item_count";
+    }
+}
diff --git a/server/src/main/java/org/opensearch/common/cache/stats/ImmutableCacheStatsHolder.java b/server/src/main/java/org/opensearch/common/cache/stats/ImmutableCacheStatsHolder.java
new file mode 100644
index 0000000000000..92383626236b8
--- /dev/null
+++ b/server/src/main/java/org/opensearch/common/cache/stats/ImmutableCacheStatsHolder.java
@@ -0,0 +1,370 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.cache.stats;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.core.common.io.stream.Writeable;
+import org.opensearch.core.xcontent.ToXContent;
+import org.opensearch.core.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.SortedMap;
+import java.util.Stack;
+import java.util.TreeMap;
+
+/**
+ * An object storing an immutable snapshot of an entire cache's stats. Accessible outside the cache itself.
+ *
+ * @opensearch.experimental
+ */
+
+@ExperimentalApi
+public class ImmutableCacheStatsHolder implements Writeable, ToXContent {
+    // Root node of immutable snapshot of stats within a CacheStatsHolder, containing all the stats maintained by the cache.
+    // Pkg-private for testing.
+    final Node statsRoot;
+    // The dimension names for each level in this tree.
+    final List<String> dimensionNames;
+    // The name of the cache type producing these stats. Returned in API response.
+    final String storeName;
+    public static String STORE_NAME_FIELD = "store_name";
+
+    // Values used for serializing/deserializing the tree.
+    private static final String SERIALIZATION_CHILDREN_OPEN_BRACKET = "<";
+    private static final String SERIALIZATION_CHILDREN_CLOSE_BRACKET = ">";
+    private static final String SERIALIZATION_BEGIN_NODE = "_";
+    private static final String SERIALIZATION_DONE = "end";
+
+    ImmutableCacheStatsHolder(
+        DefaultCacheStatsHolder.Node originalStatsRoot,
+        String[] levels,
+        List<String> originalDimensionNames,
+        String storeName
+    ) {
+        // Aggregate from the original CacheStatsHolder according to the levels passed in.
+        // The dimension names for this immutable snapshot should reflect the levels we aggregate in the snapshot
+        this.dimensionNames = filterLevels(levels, originalDimensionNames);
+        this.storeName = storeName;
+        this.statsRoot = aggregateByLevels(originalStatsRoot, originalDimensionNames);
+        makeNodeUnmodifiable(statsRoot);
+    }
+
+    public ImmutableCacheStatsHolder(StreamInput in) throws IOException {
+        this.dimensionNames = List.of(in.readStringArray());
+        this.storeName = in.readString();
+        this.statsRoot = deserializeTree(in);
+        makeNodeUnmodifiable(statsRoot);
+    }
+
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeStringArray(dimensionNames.toArray(new String[0]));
+        out.writeString(storeName);
+        writeNode(statsRoot, out);
+        out.writeString(SERIALIZATION_DONE);
+    }
+
+    private void writeNode(Node node, StreamOutput out) throws IOException {
+        out.writeString(SERIALIZATION_BEGIN_NODE);
+        out.writeString(node.dimensionValue);
+        out.writeBoolean(node.children.isEmpty()); // Write whether this is a leaf node
+        node.stats.writeTo(out);
+
+        out.writeString(SERIALIZATION_CHILDREN_OPEN_BRACKET);
+        for (Map.Entry<String, Node> entry : node.children.entrySet()) {
+            out.writeString(entry.getKey());
+            writeNode(entry.getValue(), out);
+        }
+        out.writeString(SERIALIZATION_CHILDREN_CLOSE_BRACKET);
+    }
+
+    private Node deserializeTree(StreamInput in) throws IOException {
+        final Stack<Node> stack = new Stack<>();
+        in.readString(); // Read and discard SERIALIZATION_BEGIN_NODE for the root node
+        Node statsRoot = readSingleNode(in);
+        Node current = statsRoot;
+        stack.push(statsRoot);
+        String nextSymbol = in.readString();
+        while (!nextSymbol.equals(SERIALIZATION_DONE)) {
+            switch (nextSymbol) {
+                case SERIALIZATION_CHILDREN_OPEN_BRACKET:
+                    stack.push(current);
+                    break;
+                case SERIALIZATION_CHILDREN_CLOSE_BRACKET:
+                    stack.pop();
+                    break;
+                case SERIALIZATION_BEGIN_NODE:
+                    current = readSingleNode(in);
+                    stack.peek().children.put(current.dimensionValue, current);
+            }
+            nextSymbol = in.readString();
+        }
+        return statsRoot;
+    }
+
+    private Node readSingleNode(StreamInput in) throws IOException {
+        String dimensionValue = in.readString();
+        boolean isLeafNode = in.readBoolean();
+        ImmutableCacheStats stats = new ImmutableCacheStats(in);
+        return new Node(dimensionValue, isLeafNode, stats);
+    }
+
+    private void makeNodeUnmodifiable(Node node) {
+        if (!node.children.isEmpty()) {
+            node.children = Collections.unmodifiableSortedMap(node.children);
+        }
+        for (Node child : node.children.values()) {
+            makeNodeUnmodifiable(child);
+        }
+    }
+
+    public ImmutableCacheStats getTotalStats() {
+        return statsRoot.getStats();
+    }
+
+    public long getTotalHits() {
+        return getTotalStats().getHits();
+    }
+
+    public long getTotalMisses() {
+        return getTotalStats().getMisses();
+    }
+
+    public long getTotalEvictions() {
+        return getTotalStats().getEvictions();
+    }
+
+    public long getTotalSizeInBytes() {
+        return getTotalStats().getSizeInBytes();
+    }
+
+    public long getTotalItems() {
+        return getTotalStats().getItems();
+    }
+
+    public ImmutableCacheStats getStatsForDimensionValues(List<String> dimensionValues) {
+        Node current = statsRoot;
+        for (String dimensionValue : dimensionValues) {
+            current = current.children.get(dimensionValue);
+            if (current == null) {
+                return null;
+            }
+        }
+        return current.stats;
+    }
+
+    /**
+     * Returns a new tree containing the stats aggregated by the levels passed in.
+     * The new tree only has dimensions matching the levels passed in.
+     * The levels passed in must be in the proper order, as they would be in the output of filterLevels().
+     */
+    Node aggregateByLevels(DefaultCacheStatsHolder.Node originalStatsRoot, List<String> originalDimensionNames) {
+        Node newRoot = new Node("", false, originalStatsRoot.getImmutableStats());
+        for (DefaultCacheStatsHolder.Node child : originalStatsRoot.children.values()) {
+            aggregateByLevelsHelper(newRoot, child, originalDimensionNames, 0);
+        }
+        return newRoot;
+    }
+
+    /**
+     * Because we may have to combine nodes that have the same dimension name, I don't think there's a clean way to aggregate
+     * fully recursively while also passing in a completed map of children nodes before constructing the parent node.
+     * For this reason, in this function we have to build the new tree top down rather than bottom up.
+     * We use private methods allowing us to add children to/increment the stats for an existing node.
+     * This should be ok because the resulting tree is unmodifiable after creation in the constructor.
+     *
+     * @param allDimensions the list of all dimensions present in the original CacheStatsHolder which produced
+     *                      the CacheStatsHolder.Node object we are traversing.
+     */
+    private void aggregateByLevelsHelper(
+        Node parentInNewTree,
+        DefaultCacheStatsHolder.Node currentInOriginalTree,
+        List<String> allDimensions,
+        int depth
+    ) {
+        if (dimensionNames.contains(allDimensions.get(depth))) {
+            // If this node is in a level we want to aggregate, create a new dimension node with the same value and stats, and connect it to
+            // the last parent node in the new tree. If it already exists, increment it instead.
+            String dimensionValue = currentInOriginalTree.getDimensionValue();
+            Node nodeInNewTree = parentInNewTree.children.get(dimensionValue);
+            if (nodeInNewTree == null) {
+                // Create new node with stats matching the node from the original tree
+                int indexOfLastLevel = allDimensions.indexOf(dimensionNames.get(dimensionNames.size() - 1));
+                boolean isLeafNode = depth == indexOfLastLevel; // If this is the last level we aggregate, the new node should be a leaf
+                // node
+                nodeInNewTree = new Node(dimensionValue, isLeafNode, currentInOriginalTree.getImmutableStats());
+                parentInNewTree.addChild(dimensionValue, nodeInNewTree);
+            } else {
+                // Otherwise increment existing stats
+                nodeInNewTree.incrementStats(currentInOriginalTree.getImmutableStats());
+            }
+            // Finally set the parent node to be this node for the next callers of this function
+            parentInNewTree = nodeInNewTree;
+        }
+
+        for (Map.Entry<String, DefaultCacheStatsHolder.Node> childEntry : currentInOriginalTree.children.entrySet()) {
+            DefaultCacheStatsHolder.Node child = childEntry.getValue();
+            aggregateByLevelsHelper(parentInNewTree, child, allDimensions, depth + 1);
+        }
+    }
+
+    /**
+     * Filters out levels that aren't in dimensionNames, and orders the resulting list to match the order in dimensionNames.
+     * Unrecognized levels are ignored.
+     */
+    private List<String> filterLevels(String[] levels, List<String> originalDimensionNames) {
+        if (levels == null) {
+            return originalDimensionNames;
+        }
+        List<String> levelsList = Arrays.asList(levels);
+        List<String> filtered = new ArrayList<>();
+        for (String dimensionName : originalDimensionNames) {
+            if (levelsList.contains(dimensionName)) {
+                filtered.add(dimensionName);
+            }
+        }
+        return filtered;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        // Always show total stats, regardless of levels
+        getTotalStats().toXContent(builder, params);
+
+        List<String> filteredLevels = filterLevels(getLevels(params), dimensionNames);
+        assert filteredLevels.equals(dimensionNames);
+        if (!filteredLevels.isEmpty()) {
+            // Depth -1 corresponds to the dummy root node
+            toXContentForLevels(-1, statsRoot, builder, params);
+        }
+
+        // Also add the store name for the cache that produced the stats
+        builder.field(STORE_NAME_FIELD, storeName);
+        return builder;
+    }
+
+    private void toXContentForLevels(int depth, Node current, XContentBuilder builder, Params params) throws IOException {
+        if (depth >= 0) {
+            builder.startObject(current.dimensionValue);
+        }
+
+        if (depth == dimensionNames.size() - 1) {
+            // This is a leaf node
+            current.getStats().toXContent(builder, params);
+        } else {
+            builder.startObject(dimensionNames.get(depth + 1));
+            for (Node nextNode : current.children.values()) {
+                toXContentForLevels(depth + 1, nextNode, builder, params);
+            }
+            builder.endObject();
+        }
+
+        if (depth >= 0) {
+            builder.endObject();
+        }
+    }
+
+    private String[] getLevels(Params params) {
+        String levels = params.param("level");
+        if (levels == null) {
+            return null;
+        }
+        return levels.split(",");
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (o == null || o.getClass() != ImmutableCacheStatsHolder.class) {
+            return false;
+        }
+        ImmutableCacheStatsHolder other = (ImmutableCacheStatsHolder) o;
+        if (!dimensionNames.equals(other.dimensionNames) || !storeName.equals(other.storeName)) {
+            return false;
+        }
+        return equalsHelper(statsRoot, other.getStatsRoot());
+    }
+
+    private boolean equalsHelper(Node thisNode, Node otherNode) {
+        if (otherNode == null) {
+            return false;
+        }
+        if (!thisNode.getStats().equals(otherNode.getStats())) {
+            return false;
+        }
+        boolean allChildrenMatch = true;
+        for (String childValue : thisNode.getChildren().keySet()) {
+            allChildrenMatch = equalsHelper(thisNode.children.get(childValue), otherNode.children.get(childValue));
+            if (!allChildrenMatch) {
+                return false;
+            }
+        }
+        return allChildrenMatch;
+    }
+
+    @Override
+    public int hashCode() {
+        // Should be sufficient to hash based on the total stats value (found in the root node)
+        return Objects.hash(statsRoot.stats, dimensionNames);
+    }
+
+    // A similar class to CacheStatsHolder.Node, which uses a SortedMap and holds immutable CacheStatsSnapshot as its stats.
+    static class Node {
+        private final String dimensionValue;
+        // Map from dimensionValue to the Node for that dimension value. Not final so we can set it to be unmodifiable before we are done in
+        // the constructor.
+        SortedMap<String, Node> children;
+
+        // The stats for this node. If a leaf node, corresponds to the stats for this combination of dimensions; if not,
+        // contains the sum of its children's stats.
+        private ImmutableCacheStats stats;
+        private static final SortedMap<String, Node> EMPTY_CHILDREN_MAP = Collections.unmodifiableSortedMap(new TreeMap<>());
+
+        private Node(String dimensionValue, boolean isLeafNode, ImmutableCacheStats stats) {
+            this.dimensionValue = dimensionValue;
+            this.stats = stats;
+            if (isLeafNode) {
+                this.children = EMPTY_CHILDREN_MAP;
+            } else {
+                this.children = new TreeMap<>();
+            }
+        }
+
+        Map<String, Node> getChildren() {
+            return children;
+        }
+
+        public ImmutableCacheStats getStats() {
+            return stats;
+        }
+
+        public String getDimensionValue() {
+            return dimensionValue;
+        }
+
+        private void addChild(String dimensionValue, Node child) {
+            this.children.putIfAbsent(dimensionValue, child);
+        }
+
+        private void incrementStats(ImmutableCacheStats toIncrement) {
+            stats = ImmutableCacheStats.addSnapshots(stats, toIncrement);
+        }
+    }
+
+    // pkg-private for testing
+    Node getStatsRoot() {
+        return statsRoot;
+    }
+}
diff --git a/server/src/main/java/org/opensearch/common/cache/stats/NoopCacheStatsHolder.java b/server/src/main/java/org/opensearch/common/cache/stats/NoopCacheStatsHolder.java
new file mode 100644
index 0000000000000..9cb69a3a0a365
--- /dev/null
+++ b/server/src/main/java/org/opensearch/common/cache/stats/NoopCacheStatsHolder.java
@@ -0,0 +1,69 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.cache.stats;
+
+import java.util.List;
+
+/**
+ * A dummy version of CacheStatsHolder, which cache implementations use when FeatureFlags.PLUGGABLE_CACHES is false.
+ * Returns all-zero stats when calling getImmutableCacheStatsHolder(). Always returns 0 for count().
+ * A singleton instance is used for memory purposes.
+ */
+public class NoopCacheStatsHolder implements CacheStatsHolder {
+    private static final String dummyStoreName = "noop_store";
+    private static final NoopCacheStatsHolder singletonInstance = new NoopCacheStatsHolder();
+    private static final ImmutableCacheStatsHolder immutableCacheStatsHolder;
+    static {
+        DefaultCacheStatsHolder.Node dummyNode = new DefaultCacheStatsHolder.Node("", false);
+        immutableCacheStatsHolder = new ImmutableCacheStatsHolder(dummyNode, new String[0], List.of(), dummyStoreName);
+    }
+
+    private NoopCacheStatsHolder() {}
+
+    public static NoopCacheStatsHolder getInstance() {
+        return singletonInstance;
+    }
+
+    @Override
+    public void incrementHits(List<String> dimensionValues) {}
+
+    @Override
+    public void incrementMisses(List<String> dimensionValues) {}
+
+    @Override
+    public void incrementEvictions(List<String> dimensionValues) {}
+
+    @Override
+    public void incrementSizeInBytes(List<String> dimensionValues, long amountBytes) {}
+
+    @Override
+    public void decrementSizeInBytes(List<String> dimensionValues, long amountBytes) {}
+
+    @Override
+    public void incrementItems(List<String> dimensionValues) {}
+
+    @Override
+    public void decrementItems(List<String> dimensionValues) {}
+
+    @Override
+    public void reset() {}
+
+    @Override
+    public long count() {
+        return 0;
+    }
+
+    @Override
+    public void removeDimensions(List<String> dimensionValues) {}
+
+    @Override
+    public ImmutableCacheStatsHolder getImmutableCacheStatsHolder(String[] levels) {
+        return immutableCacheStatsHolder;
+    }
+}
diff --git a/server/src/main/java/org/opensearch/common/cache/stats/package-info.java b/server/src/main/java/org/opensearch/common/cache/stats/package-info.java
new file mode 100644
index 0000000000000..95b5bc8efb510
--- /dev/null
+++ b/server/src/main/java/org/opensearch/common/cache/stats/package-info.java
@@ -0,0 +1,9 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+/** A package for cache stats. */
+package org.opensearch.common.cache.stats;
diff --git a/server/src/main/java/org/opensearch/common/cache/store/OpenSearchOnHeapCache.java b/server/src/main/java/org/opensearch/common/cache/store/OpenSearchOnHeapCache.java
index c9bec4ba47def..569653bec2a3d 100644
--- a/server/src/main/java/org/opensearch/common/cache/store/OpenSearchOnHeapCache.java
+++ b/server/src/main/java/org/opensearch/common/cache/store/OpenSearchOnHeapCache.java
@@ -12,10 +12,16 @@
 import org.opensearch.common.cache.CacheBuilder;
 import org.opensearch.common.cache.CacheType;
 import org.opensearch.common.cache.ICache;
+import org.opensearch.common.cache.ICacheKey;
 import org.opensearch.common.cache.LoadAwareCacheLoader;
 import org.opensearch.common.cache.RemovalListener;
 import org.opensearch.common.cache.RemovalNotification;
+import org.opensearch.common.cache.RemovalReason;
 import org.opensearch.common.cache.settings.CacheSettings;
+import org.opensearch.common.cache.stats.CacheStatsHolder;
+import org.opensearch.common.cache.stats.DefaultCacheStatsHolder;
+import org.opensearch.common.cache.stats.ImmutableCacheStatsHolder;
+import org.opensearch.common.cache.stats.NoopCacheStatsHolder;
 import org.opensearch.common.cache.store.builders.ICacheBuilder;
 import org.opensearch.common.cache.store.config.CacheConfig;
 import org.opensearch.common.cache.store.settings.OpenSearchOnHeapCacheSettings;
@@ -25,7 +31,10 @@
 import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.core.common.unit.ByteSizeValue;
 
+import java.util.List;
 import java.util.Map;
+import java.util.Objects;
+import java.util.function.ToLongBiFunction;
 
 import static org.opensearch.common.cache.store.settings.OpenSearchOnHeapCacheSettings.EXPIRE_AFTER_ACCESS_KEY;
 import static org.opensearch.common.cache.store.settings.OpenSearchOnHeapCacheSettings.MAXIMUM_SIZE_IN_BYTES_KEY;
@@ -37,13 +46,17 @@
  *
  * @opensearch.experimental
  */
-public class OpenSearchOnHeapCache<K, V> implements ICache<K, V>, RemovalListener<K, V> {
+public class OpenSearchOnHeapCache<K, V> implements ICache<K, V>, RemovalListener<ICacheKey<K>, V> {
 
-    private final Cache<K, V> cache;
-    private final RemovalListener<K, V> removalListener;
+    private final Cache<ICacheKey<K>, V> cache;
+    private final CacheStatsHolder cacheStatsHolder;
+    private final RemovalListener<ICacheKey<K>, V> removalListener;
+    private final List<String> dimensionNames;
+    private final ToLongBiFunction<ICacheKey<K>, V> weigher;
+    private final boolean statsTrackingEnabled;
 
     public OpenSearchOnHeapCache(Builder<K, V> builder) {
-        CacheBuilder<K, V> cacheBuilder = CacheBuilder.<K, V>builder()
+        CacheBuilder<ICacheKey<K>, V> cacheBuilder = CacheBuilder.<ICacheKey<K>, V>builder()
             .setMaximumWeight(builder.getMaxWeightInBytes())
             .weigher(builder.getWeigher())
             .removalListener(this);
@@ -51,38 +64,66 @@ public OpenSearchOnHeapCache(Builder<K, V> builder) {
             cacheBuilder.setExpireAfterAccess(builder.getExpireAfterAcess());
         }
         cache = cacheBuilder.build();
+        this.dimensionNames = Objects.requireNonNull(builder.dimensionNames, "Dimension names can't be null");
+        this.statsTrackingEnabled = builder.getStatsTrackingEnabled();
+        if (statsTrackingEnabled) {
+            this.cacheStatsHolder = new DefaultCacheStatsHolder(dimensionNames, OpenSearchOnHeapCacheFactory.NAME);
+        } else {
+            this.cacheStatsHolder = NoopCacheStatsHolder.getInstance();
+        }
         this.removalListener = builder.getRemovalListener();
+        this.weigher = builder.getWeigher();
     }
 
     @Override
-    public V get(K key) {
+    public V get(ICacheKey<K> key) {
         V value = cache.get(key);
+        if (value != null) {
+            cacheStatsHolder.incrementHits(key.dimensions);
+        } else {
+            cacheStatsHolder.incrementMisses(key.dimensions);
+        }
         return value;
     }
 
     @Override
-    public void put(K key, V value) {
+    public void put(ICacheKey<K> key, V value) {
         cache.put(key, value);
+        cacheStatsHolder.incrementItems(key.dimensions);
+        cacheStatsHolder.incrementSizeInBytes(key.dimensions, weigher.applyAsLong(key, value));
     }
 
     @Override
-    public V computeIfAbsent(K key, LoadAwareCacheLoader<K, V> loader) throws Exception {
+    public V computeIfAbsent(ICacheKey<K> key, LoadAwareCacheLoader<ICacheKey<K>, V> loader) throws Exception {
         V value = cache.computeIfAbsent(key, key1 -> loader.load(key));
+        if (!loader.isLoaded()) {
+            cacheStatsHolder.incrementHits(key.dimensions);
+        } else {
+            cacheStatsHolder.incrementMisses(key.dimensions);
+            cacheStatsHolder.incrementItems(key.dimensions);
+            cacheStatsHolder.incrementSizeInBytes(key.dimensions, cache.getWeigher().applyAsLong(key, value));
+        }
         return value;
     }
 
     @Override
-    public void invalidate(K key) {
-        cache.invalidate(key);
+    public void invalidate(ICacheKey<K> key) {
+        if (key.getDropStatsForDimensions()) {
+            cacheStatsHolder.removeDimensions(key.dimensions);
+        }
+        if (key.key != null) {
+            cache.invalidate(key);
+        }
     }
 
     @Override
     public void invalidateAll() {
         cache.invalidateAll();
+        cacheStatsHolder.reset();
     }
 
     @Override
-    public Iterable<K> keys() {
+    public Iterable<ICacheKey<K>> keys() {
         return cache.keys();
     }
 
@@ -100,8 +141,23 @@ public void refresh() {
     public void close() {}
 
     @Override
-    public void onRemoval(RemovalNotification<K, V> notification) {
-        this.removalListener.onRemoval(notification);
+    public ImmutableCacheStatsHolder stats(String[] levels) {
+        return cacheStatsHolder.getImmutableCacheStatsHolder(levels);
+    }
+
+    @Override
+    public void onRemoval(RemovalNotification<ICacheKey<K>, V> notification) {
+        removalListener.onRemoval(notification);
+        cacheStatsHolder.decrementItems(notification.getKey().dimensions);
+        cacheStatsHolder.decrementSizeInBytes(
+            notification.getKey().dimensions,
+            cache.getWeigher().applyAsLong(notification.getKey(), notification.getValue())
+        );
+
+        if (RemovalReason.EVICTED.equals(notification.getRemovalReason())
+            || RemovalReason.CAPACITY.equals(notification.getRemovalReason())) {
+            cacheStatsHolder.incrementEvictions(notification.getKey().dimensions);
+        }
     }
 
     /**
@@ -115,9 +171,10 @@ public static class OpenSearchOnHeapCacheFactory implements Factory {
         public <K, V> ICache<K, V> create(CacheConfig<K, V> config, CacheType cacheType, Map<String, Factory> cacheFactories) {
             Map<String, Setting<?>> settingList = OpenSearchOnHeapCacheSettings.getSettingListForCacheType(cacheType);
             Settings settings = config.getSettings();
-            ICacheBuilder<K, V> builder = new Builder<K, V>().setMaximumWeightInBytes(
-                ((ByteSizeValue) settingList.get(MAXIMUM_SIZE_IN_BYTES_KEY).get(settings)).getBytes()
-            )
+            boolean statsTrackingEnabled = statsTrackingEnabled(config.getSettings(), config.getStatsTrackingEnabled());
+            ICacheBuilder<K, V> builder = new Builder<K, V>().setDimensionNames(config.getDimensionNames())
+                .setStatsTrackingEnabled(statsTrackingEnabled)
+                .setMaximumWeightInBytes(((ByteSizeValue) settingList.get(MAXIMUM_SIZE_IN_BYTES_KEY).get(settings)).getBytes())
                 .setExpireAfterAccess(((TimeValue) settingList.get(EXPIRE_AFTER_ACCESS_KEY).get(settings)))
                 .setWeigher(config.getWeigher())
                 .setRemovalListener(config.getRemovalListener());
@@ -137,6 +194,11 @@ public <K, V> ICache<K, V> create(CacheConfig<K, V> config, CacheType cacheType,
         public String getCacheName() {
             return NAME;
         }
+
+        private boolean statsTrackingEnabled(Settings settings, boolean statsTrackingEnabledConfig) {
+            // Don't track stats when pluggable caching is off, or when explicitly set to false in the CacheConfig
+            return FeatureFlags.PLUGGABLE_CACHE_SETTING.get(settings) && statsTrackingEnabledConfig;
+        }
     }
 
     /**
@@ -145,6 +207,12 @@ public String getCacheName() {
      * @param <V> Type of value
      */
     public static class Builder<K, V> extends ICacheBuilder<K, V> {
+        private List<String> dimensionNames;
+
+        public Builder<K, V> setDimensionNames(List<String> dimensionNames) {
+            this.dimensionNames = dimensionNames;
+            return this;
+        }
 
         @Override
         public ICache<K, V> build() {
diff --git a/server/src/main/java/org/opensearch/common/cache/store/builders/ICacheBuilder.java b/server/src/main/java/org/opensearch/common/cache/store/builders/ICacheBuilder.java
index 7ca9080ec1aa6..a308d1db88258 100644
--- a/server/src/main/java/org/opensearch/common/cache/store/builders/ICacheBuilder.java
+++ b/server/src/main/java/org/opensearch/common/cache/store/builders/ICacheBuilder.java
@@ -10,6 +10,7 @@
 
 import org.opensearch.common.annotation.ExperimentalApi;
 import org.opensearch.common.cache.ICache;
+import org.opensearch.common.cache.ICacheKey;
 import org.opensearch.common.cache.RemovalListener;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
@@ -28,13 +29,15 @@ public abstract class ICacheBuilder<K, V> {
 
     private long maxWeightInBytes;
 
-    private ToLongBiFunction<K, V> weigher;
+    private ToLongBiFunction<ICacheKey<K>, V> weigher;
 
     private TimeValue expireAfterAcess;
 
     private Settings settings;
 
-    private RemovalListener<K, V> removalListener;
+    private RemovalListener<ICacheKey<K>, V> removalListener;
+
+    private boolean statsTrackingEnabled = true;
 
     public ICacheBuilder() {}
 
@@ -43,7 +46,7 @@ public ICacheBuilder<K, V> setMaximumWeightInBytes(long sizeInBytes) {
         return this;
     }
 
-    public ICacheBuilder<K, V> setWeigher(ToLongBiFunction<K, V> weigher) {
+    public ICacheBuilder<K, V> setWeigher(ToLongBiFunction<ICacheKey<K>, V> weigher) {
         this.weigher = weigher;
         return this;
     }
@@ -58,11 +61,16 @@ public ICacheBuilder<K, V> setSettings(Settings settings) {
         return this;
     }
 
-    public ICacheBuilder<K, V> setRemovalListener(RemovalListener<K, V> removalListener) {
+    public ICacheBuilder<K, V> setRemovalListener(RemovalListener<ICacheKey<K>, V> removalListener) {
         this.removalListener = removalListener;
         return this;
     }
 
+    public ICacheBuilder<K, V> setStatsTrackingEnabled(boolean statsTrackingEnabled) {
+        this.statsTrackingEnabled = statsTrackingEnabled;
+        return this;
+    }
+
     public long getMaxWeightInBytes() {
         return maxWeightInBytes;
     }
@@ -71,11 +79,11 @@ public TimeValue getExpireAfterAcess() {
         return expireAfterAcess;
     }
 
-    public ToLongBiFunction<K, V> getWeigher() {
+    public ToLongBiFunction<ICacheKey<K>, V> getWeigher() {
         return weigher;
     }
 
-    public RemovalListener<K, V> getRemovalListener() {
+    public RemovalListener<ICacheKey<K>, V> getRemovalListener() {
         return this.removalListener;
     }
 
@@ -83,5 +91,9 @@ public Settings getSettings() {
         return settings;
     }
 
+    public boolean getStatsTrackingEnabled() {
+        return statsTrackingEnabled;
+    }
+
     public abstract ICache<K, V> build();
 }
diff --git a/server/src/main/java/org/opensearch/common/cache/store/config/CacheConfig.java b/server/src/main/java/org/opensearch/common/cache/store/config/CacheConfig.java
index 4c9881e845d42..0c54ac57a9b18 100644
--- a/server/src/main/java/org/opensearch/common/cache/store/config/CacheConfig.java
+++ b/server/src/main/java/org/opensearch/common/cache/store/config/CacheConfig.java
@@ -9,12 +9,15 @@
 package org.opensearch.common.cache.store.config;
 
 import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.common.cache.ICacheKey;
 import org.opensearch.common.cache.RemovalListener;
 import org.opensearch.common.cache.policy.CachedQueryResult;
 import org.opensearch.common.cache.serializer.Serializer;
+import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
 
+import java.util.List;
 import java.util.function.Function;
 import java.util.function.ToLongBiFunction;
 
@@ -41,9 +44,11 @@ public class CacheConfig<K, V> {
     /**
      * Represents a function that calculates the size or weight of a key-value pair.
      */
-    private final ToLongBiFunction<K, V> weigher;
+    private final ToLongBiFunction<ICacheKey<K>, V> weigher;
 
-    private final RemovalListener<K, V> removalListener;
+    private final RemovalListener<ICacheKey<K>, V> removalListener;
+
+    private final List<String> dimensionNames;
 
     // Serializers for keys and values. Not required for all caches.
     private final Serializer<K, ?> keySerializer;
@@ -61,6 +66,10 @@ public class CacheConfig<K, V> {
      */
     private final TimeValue expireAfterAccess;
 
+    private final ClusterSettings clusterSettings;
+
+    private final boolean statsTrackingEnabled;
+
     private CacheConfig(Builder<K, V> builder) {
         this.keyType = builder.keyType;
         this.valueType = builder.valueType;
@@ -69,9 +78,12 @@ private CacheConfig(Builder<K, V> builder) {
         this.weigher = builder.weigher;
         this.keySerializer = builder.keySerializer;
         this.valueSerializer = builder.valueSerializer;
+        this.dimensionNames = builder.dimensionNames;
         this.cachedResultParser = builder.cachedResultParser;
         this.maxSizeInBytes = builder.maxSizeInBytes;
         this.expireAfterAccess = builder.expireAfterAccess;
+        this.clusterSettings = builder.clusterSettings;
+        this.statsTrackingEnabled = builder.statsTrackingEnabled;
     }
 
     public Class<K> getKeyType() {
@@ -86,7 +98,7 @@ public Settings getSettings() {
         return settings;
     }
 
-    public RemovalListener<K, V> getRemovalListener() {
+    public RemovalListener<ICacheKey<K>, V> getRemovalListener() {
         return removalListener;
     }
 
@@ -98,7 +110,7 @@ public RemovalListener<K, V> getRemovalListener() {
         return valueSerializer;
     }
 
-    public ToLongBiFunction<K, V> getWeigher() {
+    public ToLongBiFunction<ICacheKey<K>, V> getWeigher() {
         return weigher;
     }
 
@@ -106,6 +118,10 @@ public Function<V, CachedQueryResult.PolicyValues> getCachedResultParser() {
         return cachedResultParser;
     }
 
+    public List<String> getDimensionNames() {
+        return dimensionNames;
+    }
+
     public Long getMaxSizeInBytes() {
         return maxSizeInBytes;
     }
@@ -114,6 +130,14 @@ public TimeValue getExpireAfterAccess() {
         return expireAfterAccess;
     }
 
+    public ClusterSettings getClusterSettings() {
+        return clusterSettings;
+    }
+
+    public boolean getStatsTrackingEnabled() {
+        return statsTrackingEnabled;
+    }
+
     /**
      * Builder class to build Cache config related parameters.
      * @param <K> Type of key.
@@ -127,17 +151,18 @@ public static class Builder<K, V> {
 
         private Class<V> valueType;
 
-        private RemovalListener<K, V> removalListener;
-
+        private RemovalListener<ICacheKey<K>, V> removalListener;
+        private List<String> dimensionNames;
         private Serializer<K, ?> keySerializer;
         private Serializer<V, ?> valueSerializer;
-
-        private ToLongBiFunction<K, V> weigher;
+        private ToLongBiFunction<ICacheKey<K>, V> weigher;
         private Function<V, CachedQueryResult.PolicyValues> cachedResultParser;
 
         private long maxSizeInBytes;
 
         private TimeValue expireAfterAccess;
+        private ClusterSettings clusterSettings;
+        private boolean statsTrackingEnabled = true;
 
         public Builder() {}
 
@@ -156,11 +181,16 @@ public Builder<K, V> setValueType(Class<V> valueType) {
             return this;
         }
 
-        public Builder<K, V> setRemovalListener(RemovalListener<K, V> removalListener) {
+        public Builder<K, V> setRemovalListener(RemovalListener<ICacheKey<K>, V> removalListener) {
             this.removalListener = removalListener;
             return this;
         }
 
+        public Builder<K, V> setWeigher(ToLongBiFunction<ICacheKey<K>, V> weigher) {
+            this.weigher = weigher;
+            return this;
+        }
+
         public Builder<K, V> setKeySerializer(Serializer<K, ?> keySerializer) {
             this.keySerializer = keySerializer;
             return this;
@@ -171,8 +201,8 @@ public Builder<K, V> setValueSerializer(Serializer<V, ?> valueSerializer) {
             return this;
         }
 
-        public Builder<K, V> setWeigher(ToLongBiFunction<K, V> weigher) {
-            this.weigher = weigher;
+        public Builder<K, V> setDimensionNames(List<String> dimensionNames) {
+            this.dimensionNames = dimensionNames;
             return this;
         }
 
@@ -191,6 +221,16 @@ public Builder<K, V> setExpireAfterAccess(TimeValue expireAfterAccess) {
             return this;
         }
 
+        public Builder<K, V> setClusterSettings(ClusterSettings clusterSettings) {
+            this.clusterSettings = clusterSettings;
+            return this;
+        }
+
+        public Builder<K, V> setStatsTrackingEnabled(boolean statsTrackingEnabled) {
+            this.statsTrackingEnabled = statsTrackingEnabled;
+            return this;
+        }
+
         public CacheConfig<K, V> build() {
             return new CacheConfig<>(this);
         }
diff --git a/server/src/main/java/org/opensearch/common/hash/FNV1a.java b/server/src/main/java/org/opensearch/common/hash/FNV1a.java
new file mode 100644
index 0000000000000..cab28d0f2d68f
--- /dev/null
+++ b/server/src/main/java/org/opensearch/common/hash/FNV1a.java
@@ -0,0 +1,48 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.hash;
+
+import java.nio.charset.StandardCharsets;
+
+/**
+ * Provides hashing function using FNV1a hash function. @see <a href="http://www.isthe.com/chongo/tech/comp/fnv/#FNV-1">FNV author's website</a>.
+ * 32 bit Java port of http://www.isthe.com/chongo/src/fnv/hash_32a.c
+ * 64 bit Java port of http://www.isthe.com/chongo/src/fnv/hash_64a.c
+ *
+ * @opensearch.internal
+ */
+public class FNV1a {
+    private static final long FNV_OFFSET_BASIS_32 = 0x811c9dc5L;
+    private static final long FNV_PRIME_32 = 0x01000193L;
+
+    private static final long FNV_OFFSET_BASIS_64 = 0xcbf29ce484222325L;
+    private static final long FNV_PRIME_64 = 0x100000001b3L;
+
+    // FNV-1a hash computation for 32-bit hash
+    public static long hash32(String input) {
+        long hash = FNV_OFFSET_BASIS_32;
+        byte[] bytes = input.getBytes(StandardCharsets.UTF_8);
+        for (byte b : bytes) {
+            hash ^= (b & 0xFF);
+            hash *= FNV_PRIME_32;
+        }
+        return hash;
+    }
+
+    // FNV-1a hash computation for 64-bit hash
+    public static long hash64(String input) {
+        long hash = FNV_OFFSET_BASIS_64;
+        byte[] bytes = input.getBytes(StandardCharsets.UTF_8);
+        for (byte b : bytes) {
+            hash ^= (b & 0xFF);
+            hash *= FNV_PRIME_64;
+        }
+        return hash;
+    }
+}
diff --git a/server/src/main/java/org/opensearch/common/inject/Binder.java b/server/src/main/java/org/opensearch/common/inject/Binder.java
index a733a19608ac1..a9d16becfb5ab 100644
--- a/server/src/main/java/org/opensearch/common/inject/Binder.java
+++ b/server/src/main/java/org/opensearch/common/inject/Binder.java
@@ -29,6 +29,7 @@
 
 package org.opensearch.common.inject;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.inject.binder.AnnotatedBindingBuilder;
 import org.opensearch.common.inject.binder.AnnotatedConstantBindingBuilder;
 import org.opensearch.common.inject.binder.LinkedBindingBuilder;
@@ -198,8 +199,9 @@
  * @author jessewilson@google.com (Jesse Wilson)
  * @author kevinb@google.com (Kevin Bourrillion)
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface Binder {
 
     /**
diff --git a/server/src/main/java/org/opensearch/common/inject/Binding.java b/server/src/main/java/org/opensearch/common/inject/Binding.java
index 53d02e37502af..a42237697a1d2 100644
--- a/server/src/main/java/org/opensearch/common/inject/Binding.java
+++ b/server/src/main/java/org/opensearch/common/inject/Binding.java
@@ -29,6 +29,7 @@
 
 package org.opensearch.common.inject;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.inject.spi.BindingScopingVisitor;
 import org.opensearch.common.inject.spi.BindingTargetVisitor;
 import org.opensearch.common.inject.spi.Element;
@@ -69,8 +70,9 @@
  * @author crazybob@google.com (Bob Lee)
  * @author jessewilson@google.com (Jesse Wilson)
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface Binding<T> extends Element {
 
     /**
diff --git a/server/src/main/java/org/opensearch/common/inject/ConfigurationException.java b/server/src/main/java/org/opensearch/common/inject/ConfigurationException.java
index 4379a93482560..e3a32754a1bdb 100644
--- a/server/src/main/java/org/opensearch/common/inject/ConfigurationException.java
+++ b/server/src/main/java/org/opensearch/common/inject/ConfigurationException.java
@@ -29,6 +29,7 @@
 
 package org.opensearch.common.inject;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.inject.internal.Errors;
 import org.opensearch.common.inject.spi.Message;
 
@@ -46,8 +47,9 @@
  * @author jessewilson@google.com (Jesse Wilson)
  * @since 2.0
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class ConfigurationException extends RuntimeException {
     private final Set<Message> messages;
     private Object partialValue = null;
diff --git a/server/src/main/java/org/opensearch/common/inject/Injector.java b/server/src/main/java/org/opensearch/common/inject/Injector.java
index ff212c6313371..772578dd6bb2c 100644
--- a/server/src/main/java/org/opensearch/common/inject/Injector.java
+++ b/server/src/main/java/org/opensearch/common/inject/Injector.java
@@ -29,6 +29,8 @@
 
 package org.opensearch.common.inject;
 
+import org.opensearch.common.annotation.PublicApi;
+
 import java.util.List;
 
 /**
@@ -54,8 +56,9 @@
  * @author crazybob@google.com (Bob Lee)
  * @author jessewilson@google.com (Jesse Wilson)
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface Injector {
 
     /**
diff --git a/server/src/main/java/org/opensearch/common/inject/Key.java b/server/src/main/java/org/opensearch/common/inject/Key.java
index cd305353a555d..32f168d18e523 100644
--- a/server/src/main/java/org/opensearch/common/inject/Key.java
+++ b/server/src/main/java/org/opensearch/common/inject/Key.java
@@ -29,6 +29,7 @@
 
 package org.opensearch.common.inject;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.inject.internal.Annotations;
 import org.opensearch.common.inject.internal.MoreTypes;
 import org.opensearch.common.inject.internal.ToStringBuilder;
@@ -59,8 +60,9 @@
  *
  * @author crazybob@google.com (Bob Lee)
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class Key<T> {
 
     private final AnnotationStrategy annotationStrategy;
diff --git a/server/src/main/java/org/opensearch/common/inject/MembersInjector.java b/server/src/main/java/org/opensearch/common/inject/MembersInjector.java
index 891762375d5a2..872ae883e246b 100644
--- a/server/src/main/java/org/opensearch/common/inject/MembersInjector.java
+++ b/server/src/main/java/org/opensearch/common/inject/MembersInjector.java
@@ -29,6 +29,8 @@
 
 package org.opensearch.common.inject;
 
+import org.opensearch.common.annotation.PublicApi;
+
 /**
  * Injects dependencies into the fields and methods on instances of type {@code T}. Ignores the
  * presence or absence of an injectable constructor.
@@ -38,8 +40,9 @@
  * @author jessewilson@google.com (Jesse Wilson)
  * @since 2.0
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface MembersInjector<T> {
 
     /**
diff --git a/server/src/main/java/org/opensearch/common/inject/Module.java b/server/src/main/java/org/opensearch/common/inject/Module.java
index b1fc031192ea0..e66044ff26c40 100644
--- a/server/src/main/java/org/opensearch/common/inject/Module.java
+++ b/server/src/main/java/org/opensearch/common/inject/Module.java
@@ -29,6 +29,8 @@
 
 package org.opensearch.common.inject;
 
+import org.opensearch.common.annotation.PublicApi;
+
 /**
  * A module contributes configuration information, typically interface
  * bindings, which will be used to create an {@link Injector}. A Guice-based
@@ -43,8 +45,9 @@
  * Use scope and binding annotations on these methods to configure the
  * bindings.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface Module {
 
     /**
diff --git a/server/src/main/java/org/opensearch/common/inject/PrivateBinder.java b/server/src/main/java/org/opensearch/common/inject/PrivateBinder.java
index 87635880e29d8..2b6b2e0aad146 100644
--- a/server/src/main/java/org/opensearch/common/inject/PrivateBinder.java
+++ b/server/src/main/java/org/opensearch/common/inject/PrivateBinder.java
@@ -29,6 +29,7 @@
 
 package org.opensearch.common.inject;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.inject.binder.AnnotatedElementBuilder;
 
 /**
@@ -38,8 +39,9 @@
  * @author jessewilson@google.com (Jesse Wilson)
  * @since 2.0
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface PrivateBinder extends Binder {
 
     /**
diff --git a/server/src/main/java/org/opensearch/common/inject/Provider.java b/server/src/main/java/org/opensearch/common/inject/Provider.java
index 97f9e9ae503cd..988143b328828 100644
--- a/server/src/main/java/org/opensearch/common/inject/Provider.java
+++ b/server/src/main/java/org/opensearch/common/inject/Provider.java
@@ -29,6 +29,8 @@
 
 package org.opensearch.common.inject;
 
+import org.opensearch.common.annotation.PublicApi;
+
 /**
  * An object capable of providing instances of type {@code T}. Providers are used in numerous ways
  * by Guice:
@@ -50,8 +52,9 @@
  * @param <T> the type of object this provides
  * @author crazybob@google.com (Bob Lee)
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface Provider<T> {
 
     /**
diff --git a/server/src/main/java/org/opensearch/common/inject/Scope.java b/server/src/main/java/org/opensearch/common/inject/Scope.java
index a21495f522d5e..6fb9f560981ef 100644
--- a/server/src/main/java/org/opensearch/common/inject/Scope.java
+++ b/server/src/main/java/org/opensearch/common/inject/Scope.java
@@ -29,6 +29,8 @@
 
 package org.opensearch.common.inject;
 
+import org.opensearch.common.annotation.PublicApi;
+
 /**
  * A scope is a level of visibility that instances provided by Guice may have.
  * By default, an instance created by the {@link Injector} has <i>no scope</i>,
@@ -42,8 +44,9 @@
  *
  * @author crazybob@google.com (Bob Lee)
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface Scope {
 
     /**
diff --git a/server/src/main/java/org/opensearch/common/inject/Stage.java b/server/src/main/java/org/opensearch/common/inject/Stage.java
index d5996bd1363e9..fbb6e389ef43f 100644
--- a/server/src/main/java/org/opensearch/common/inject/Stage.java
+++ b/server/src/main/java/org/opensearch/common/inject/Stage.java
@@ -29,13 +29,16 @@
 
 package org.opensearch.common.inject;
 
+import org.opensearch.common.annotation.PublicApi;
+
 /**
  * The stage we're running in.
  *
  * @author crazybob@google.com (Bob Lee)
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public enum Stage {
 
     /**
diff --git a/server/src/main/java/org/opensearch/common/inject/TypeLiteral.java b/server/src/main/java/org/opensearch/common/inject/TypeLiteral.java
index f0cca2990b407..8ac04e5d0ac1d 100644
--- a/server/src/main/java/org/opensearch/common/inject/TypeLiteral.java
+++ b/server/src/main/java/org/opensearch/common/inject/TypeLiteral.java
@@ -29,6 +29,7 @@
 
 package org.opensearch.common.inject;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.inject.internal.MoreTypes;
 import org.opensearch.common.inject.util.Types;
 
@@ -77,8 +78,9 @@
  * @author crazybob@google.com (Bob Lee)
  * @author jessewilson@google.com (Jesse Wilson)
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class TypeLiteral<T> {
 
     final Class<? super T> rawType;
diff --git a/server/src/main/java/org/opensearch/common/inject/binder/AnnotatedBindingBuilder.java b/server/src/main/java/org/opensearch/common/inject/binder/AnnotatedBindingBuilder.java
index bcd593a8cbf7b..5c3c6eac9bd3a 100644
--- a/server/src/main/java/org/opensearch/common/inject/binder/AnnotatedBindingBuilder.java
+++ b/server/src/main/java/org/opensearch/common/inject/binder/AnnotatedBindingBuilder.java
@@ -29,6 +29,8 @@
 
 package org.opensearch.common.inject.binder;
 
+import org.opensearch.common.annotation.PublicApi;
+
 import java.lang.annotation.Annotation;
 
 /**
@@ -36,8 +38,9 @@
  *
  * @author crazybob@google.com (Bob Lee)
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface AnnotatedBindingBuilder<T> extends LinkedBindingBuilder<T> {
 
     /**
diff --git a/server/src/main/java/org/opensearch/common/inject/binder/AnnotatedConstantBindingBuilder.java b/server/src/main/java/org/opensearch/common/inject/binder/AnnotatedConstantBindingBuilder.java
index 42c208a2b37ea..71ea1ba0a5207 100644
--- a/server/src/main/java/org/opensearch/common/inject/binder/AnnotatedConstantBindingBuilder.java
+++ b/server/src/main/java/org/opensearch/common/inject/binder/AnnotatedConstantBindingBuilder.java
@@ -29,6 +29,8 @@
 
 package org.opensearch.common.inject.binder;
 
+import org.opensearch.common.annotation.PublicApi;
+
 import java.lang.annotation.Annotation;
 
 /**
@@ -36,8 +38,9 @@
  *
  * @author crazybob@google.com (Bob Lee)
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface AnnotatedConstantBindingBuilder {
 
     /**
diff --git a/server/src/main/java/org/opensearch/common/inject/binder/AnnotatedElementBuilder.java b/server/src/main/java/org/opensearch/common/inject/binder/AnnotatedElementBuilder.java
index f2d0916790b6b..54fcb915d83c9 100644
--- a/server/src/main/java/org/opensearch/common/inject/binder/AnnotatedElementBuilder.java
+++ b/server/src/main/java/org/opensearch/common/inject/binder/AnnotatedElementBuilder.java
@@ -29,6 +29,8 @@
 
 package org.opensearch.common.inject.binder;
 
+import org.opensearch.common.annotation.PublicApi;
+
 import java.lang.annotation.Annotation;
 
 /**
@@ -37,8 +39,9 @@
  * @author jessewilson@google.com (Jesse Wilson)
  * @since 2.0
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface AnnotatedElementBuilder {
 
     /**
diff --git a/server/src/main/java/org/opensearch/common/inject/binder/ConstantBindingBuilder.java b/server/src/main/java/org/opensearch/common/inject/binder/ConstantBindingBuilder.java
index 595c477d3e28b..feaee3ed59f46 100644
--- a/server/src/main/java/org/opensearch/common/inject/binder/ConstantBindingBuilder.java
+++ b/server/src/main/java/org/opensearch/common/inject/binder/ConstantBindingBuilder.java
@@ -29,11 +29,14 @@
 
 package org.opensearch.common.inject.binder;
 
+import org.opensearch.common.annotation.PublicApi;
+
 /**
  * Binds to a constant value.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface ConstantBindingBuilder {
 
     /**
diff --git a/server/src/main/java/org/opensearch/common/inject/binder/LinkedBindingBuilder.java b/server/src/main/java/org/opensearch/common/inject/binder/LinkedBindingBuilder.java
index 2368fef16471c..e8c4b197253b5 100644
--- a/server/src/main/java/org/opensearch/common/inject/binder/LinkedBindingBuilder.java
+++ b/server/src/main/java/org/opensearch/common/inject/binder/LinkedBindingBuilder.java
@@ -29,6 +29,7 @@
 
 package org.opensearch.common.inject.binder;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.inject.Key;
 import org.opensearch.common.inject.Provider;
 import org.opensearch.common.inject.TypeLiteral;
@@ -38,8 +39,9 @@
  *
  * @author crazybob@google.com (Bob Lee)
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface LinkedBindingBuilder<T> extends ScopedBindingBuilder {
 
     /**
diff --git a/server/src/main/java/org/opensearch/common/inject/binder/ScopedBindingBuilder.java b/server/src/main/java/org/opensearch/common/inject/binder/ScopedBindingBuilder.java
index 73dd4414f17a2..c360b9571bc4a 100644
--- a/server/src/main/java/org/opensearch/common/inject/binder/ScopedBindingBuilder.java
+++ b/server/src/main/java/org/opensearch/common/inject/binder/ScopedBindingBuilder.java
@@ -29,6 +29,7 @@
 
 package org.opensearch.common.inject.binder;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.inject.Scope;
 
 import java.lang.annotation.Annotation;
@@ -38,8 +39,9 @@
  *
  * @author crazybob@google.com (Bob Lee)
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface ScopedBindingBuilder {
 
     /**
diff --git a/server/src/main/java/org/opensearch/common/inject/matcher/Matcher.java b/server/src/main/java/org/opensearch/common/inject/matcher/Matcher.java
index 21bb63cfef097..4e254f8641350 100644
--- a/server/src/main/java/org/opensearch/common/inject/matcher/Matcher.java
+++ b/server/src/main/java/org/opensearch/common/inject/matcher/Matcher.java
@@ -29,13 +29,16 @@
 
 package org.opensearch.common.inject.matcher;
 
+import org.opensearch.common.annotation.PublicApi;
+
 /**
  * Returns {@code true} or {@code false} for a given input.
  *
  * @author crazybob@google.com (Bob Lee)
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface Matcher<T> {
 
     /**
diff --git a/server/src/main/java/org/opensearch/common/inject/spi/BindingScopingVisitor.java b/server/src/main/java/org/opensearch/common/inject/spi/BindingScopingVisitor.java
index d7c7d9d65051d..b4fbdf2fdb72b 100644
--- a/server/src/main/java/org/opensearch/common/inject/spi/BindingScopingVisitor.java
+++ b/server/src/main/java/org/opensearch/common/inject/spi/BindingScopingVisitor.java
@@ -29,6 +29,7 @@
 
 package org.opensearch.common.inject.spi;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.inject.Scope;
 
 import java.lang.annotation.Annotation;
@@ -40,8 +41,9 @@
  *            {@code return null} if no return type is needed.
  * @since 2.0
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface BindingScopingVisitor<V> {
 
     /**
diff --git a/server/src/main/java/org/opensearch/common/inject/spi/BindingTargetVisitor.java b/server/src/main/java/org/opensearch/common/inject/spi/BindingTargetVisitor.java
index 91df812b58ac4..9543e731308bd 100644
--- a/server/src/main/java/org/opensearch/common/inject/spi/BindingTargetVisitor.java
+++ b/server/src/main/java/org/opensearch/common/inject/spi/BindingTargetVisitor.java
@@ -29,6 +29,8 @@
 
 package org.opensearch.common.inject.spi;
 
+import org.opensearch.common.annotation.PublicApi;
+
 /**
  * Visits each of the strategies used to find an instance to satisfy an injection.
  *
@@ -36,8 +38,9 @@
  *            {@code return null} if no return type is needed.
  * @since 2.0
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface BindingTargetVisitor<T, V> {
 
     /**
diff --git a/server/src/main/java/org/opensearch/common/inject/spi/ConstructorBinding.java b/server/src/main/java/org/opensearch/common/inject/spi/ConstructorBinding.java
index 997bf78234fd1..8eec6cefe53c7 100644
--- a/server/src/main/java/org/opensearch/common/inject/spi/ConstructorBinding.java
+++ b/server/src/main/java/org/opensearch/common/inject/spi/ConstructorBinding.java
@@ -29,6 +29,7 @@
 
 package org.opensearch.common.inject.spi;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.inject.Binding;
 
 import java.util.Set;
@@ -40,8 +41,9 @@
  * @author jessewilson@google.com (Jesse Wilson)
  * @since 2.0
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface ConstructorBinding<T> extends Binding<T>, HasDependencies {
 
     /**
diff --git a/server/src/main/java/org/opensearch/common/inject/spi/ConvertedConstantBinding.java b/server/src/main/java/org/opensearch/common/inject/spi/ConvertedConstantBinding.java
index e8d6b346f8596..a07da68a88931 100644
--- a/server/src/main/java/org/opensearch/common/inject/spi/ConvertedConstantBinding.java
+++ b/server/src/main/java/org/opensearch/common/inject/spi/ConvertedConstantBinding.java
@@ -29,6 +29,7 @@
 
 package org.opensearch.common.inject.spi;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.inject.Binding;
 import org.opensearch.common.inject.Key;
 
@@ -41,8 +42,9 @@
  * @author jessewilson@google.com (Jesse Wilson)
  * @since 2.0
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface ConvertedConstantBinding<T> extends Binding<T>, HasDependencies {
 
     /**
diff --git a/server/src/main/java/org/opensearch/common/inject/spi/Dependency.java b/server/src/main/java/org/opensearch/common/inject/spi/Dependency.java
index be1336ad0f297..e541ba0b73bf5 100644
--- a/server/src/main/java/org/opensearch/common/inject/spi/Dependency.java
+++ b/server/src/main/java/org/opensearch/common/inject/spi/Dependency.java
@@ -29,6 +29,7 @@
 
 package org.opensearch.common.inject.spi;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.inject.Key;
 
 import java.util.HashSet;
@@ -47,8 +48,9 @@
  * @author jessewilson@google.com (Jesse Wilson)
  * @since 2.0
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class Dependency<T> {
     private final InjectionPoint injectionPoint;
     private final Key<T> key;
diff --git a/server/src/main/java/org/opensearch/common/inject/spi/Element.java b/server/src/main/java/org/opensearch/common/inject/spi/Element.java
index 660aca1bd45ab..58a696fb7ffa9 100644
--- a/server/src/main/java/org/opensearch/common/inject/spi/Element.java
+++ b/server/src/main/java/org/opensearch/common/inject/spi/Element.java
@@ -29,6 +29,7 @@
 
 package org.opensearch.common.inject.spi;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.inject.Binder;
 
 /**
@@ -43,8 +44,9 @@
  * @author crazybob@google.com (Bob Lee)
  * @since 2.0
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface Element {
 
     /**
diff --git a/server/src/main/java/org/opensearch/common/inject/spi/ElementVisitor.java b/server/src/main/java/org/opensearch/common/inject/spi/ElementVisitor.java
index d415560fc03c8..b88f11b9378aa 100644
--- a/server/src/main/java/org/opensearch/common/inject/spi/ElementVisitor.java
+++ b/server/src/main/java/org/opensearch/common/inject/spi/ElementVisitor.java
@@ -29,6 +29,7 @@
 
 package org.opensearch.common.inject.spi;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.inject.Binding;
 
 /**
@@ -38,8 +39,9 @@
  *            {@code return null} if no return type is needed.
  * @since 2.0
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface ElementVisitor<V> {
 
     /**
diff --git a/server/src/main/java/org/opensearch/common/inject/spi/ExposedBinding.java b/server/src/main/java/org/opensearch/common/inject/spi/ExposedBinding.java
index d2563bc2728cd..6c1679432abe5 100644
--- a/server/src/main/java/org/opensearch/common/inject/spi/ExposedBinding.java
+++ b/server/src/main/java/org/opensearch/common/inject/spi/ExposedBinding.java
@@ -29,6 +29,7 @@
 
 package org.opensearch.common.inject.spi;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.inject.Binder;
 import org.opensearch.common.inject.Binding;
 
@@ -38,8 +39,9 @@
  * @author jessewilson@google.com (Jesse Wilson)
  * @since 2.0
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface ExposedBinding<T> extends Binding<T>, HasDependencies {
 
     /**
diff --git a/server/src/main/java/org/opensearch/common/inject/spi/InjectionListener.java b/server/src/main/java/org/opensearch/common/inject/spi/InjectionListener.java
index 7a760d2b84e9f..878e919cda4cc 100644
--- a/server/src/main/java/org/opensearch/common/inject/spi/InjectionListener.java
+++ b/server/src/main/java/org/opensearch/common/inject/spi/InjectionListener.java
@@ -29,6 +29,8 @@
 
 package org.opensearch.common.inject.spi;
 
+import org.opensearch.common.annotation.PublicApi;
+
 /**
  * Listens for injections into instances of type {@code I}. Useful for performing further
  * injections, post-injection initialization, and more.
@@ -37,8 +39,9 @@
  * @author jessewilson@google.com (Jesse Wilson)
  * @since 2.0
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface InjectionListener<I> {
 
     /**
diff --git a/server/src/main/java/org/opensearch/common/inject/spi/InjectionPoint.java b/server/src/main/java/org/opensearch/common/inject/spi/InjectionPoint.java
index c88b2281107ed..542cbd780a8b6 100644
--- a/server/src/main/java/org/opensearch/common/inject/spi/InjectionPoint.java
+++ b/server/src/main/java/org/opensearch/common/inject/spi/InjectionPoint.java
@@ -29,6 +29,7 @@
 
 package org.opensearch.common.inject.spi;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.inject.ConfigurationException;
 import org.opensearch.common.inject.Inject;
 import org.opensearch.common.inject.Key;
@@ -66,8 +67,9 @@
  * @author crazybob@google.com (Bob Lee)
  * @since 2.0
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class InjectionPoint {
 
     private final boolean optional;
diff --git a/server/src/main/java/org/opensearch/common/inject/spi/InjectionRequest.java b/server/src/main/java/org/opensearch/common/inject/spi/InjectionRequest.java
index 6ce5febbb6711..a5faca6264424 100644
--- a/server/src/main/java/org/opensearch/common/inject/spi/InjectionRequest.java
+++ b/server/src/main/java/org/opensearch/common/inject/spi/InjectionRequest.java
@@ -29,6 +29,7 @@
 
 package org.opensearch.common.inject.spi;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.inject.Binder;
 import org.opensearch.common.inject.ConfigurationException;
 import org.opensearch.common.inject.TypeLiteral;
@@ -46,8 +47,9 @@
  * @author mikeward@google.com (Mike Ward)
  * @since 2.0
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class InjectionRequest<T> implements Element {
 
     private final Object source;
diff --git a/server/src/main/java/org/opensearch/common/inject/spi/InstanceBinding.java b/server/src/main/java/org/opensearch/common/inject/spi/InstanceBinding.java
index fd7c1303ed6fc..f73b284ae2e8c 100644
--- a/server/src/main/java/org/opensearch/common/inject/spi/InstanceBinding.java
+++ b/server/src/main/java/org/opensearch/common/inject/spi/InstanceBinding.java
@@ -29,6 +29,7 @@
 
 package org.opensearch.common.inject.spi;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.inject.Binding;
 
 import java.util.Set;
@@ -39,8 +40,9 @@
  * @author jessewilson@google.com (Jesse Wilson)
  * @since 2.0
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface InstanceBinding<T> extends Binding<T>, HasDependencies {
 
     /**
diff --git a/server/src/main/java/org/opensearch/common/inject/spi/LinkedKeyBinding.java b/server/src/main/java/org/opensearch/common/inject/spi/LinkedKeyBinding.java
index 10b270e499603..01da905f8da47 100644
--- a/server/src/main/java/org/opensearch/common/inject/spi/LinkedKeyBinding.java
+++ b/server/src/main/java/org/opensearch/common/inject/spi/LinkedKeyBinding.java
@@ -29,6 +29,7 @@
 
 package org.opensearch.common.inject.spi;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.inject.Binding;
 import org.opensearch.common.inject.Key;
 
@@ -38,8 +39,9 @@
  * @author jessewilson@google.com (Jesse Wilson)
  * @since 2.0
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface LinkedKeyBinding<T> extends Binding<T> {
 
     /**
diff --git a/server/src/main/java/org/opensearch/common/inject/spi/MembersInjectorLookup.java b/server/src/main/java/org/opensearch/common/inject/spi/MembersInjectorLookup.java
index 1f652708de875..b8a07146812c1 100644
--- a/server/src/main/java/org/opensearch/common/inject/spi/MembersInjectorLookup.java
+++ b/server/src/main/java/org/opensearch/common/inject/spi/MembersInjectorLookup.java
@@ -29,6 +29,7 @@
 
 package org.opensearch.common.inject.spi;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.inject.Binder;
 import org.opensearch.common.inject.MembersInjector;
 import org.opensearch.common.inject.TypeLiteral;
@@ -45,8 +46,9 @@
  * @author crazybob@google.com (Bob Lee)
  * @since 2.0
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class MembersInjectorLookup<T> implements Element {
 
     private final Object source;
diff --git a/server/src/main/java/org/opensearch/common/inject/spi/Message.java b/server/src/main/java/org/opensearch/common/inject/spi/Message.java
index 78829e82c150e..13184a7d82f0c 100644
--- a/server/src/main/java/org/opensearch/common/inject/spi/Message.java
+++ b/server/src/main/java/org/opensearch/common/inject/spi/Message.java
@@ -29,6 +29,7 @@
 
 package org.opensearch.common.inject.spi;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.inject.Binder;
 import org.opensearch.common.inject.internal.Errors;
 import org.opensearch.common.inject.internal.SourceProvider;
@@ -50,8 +51,9 @@
  *
  * @author crazybob@google.com (Bob Lee)
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class Message implements Element {
     private final String message;
     private final Throwable cause;
diff --git a/server/src/main/java/org/opensearch/common/inject/spi/PrivateElements.java b/server/src/main/java/org/opensearch/common/inject/spi/PrivateElements.java
index e4d86a356cd53..6330cbe33de58 100644
--- a/server/src/main/java/org/opensearch/common/inject/spi/PrivateElements.java
+++ b/server/src/main/java/org/opensearch/common/inject/spi/PrivateElements.java
@@ -29,6 +29,7 @@
 
 package org.opensearch.common.inject.spi;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.inject.Injector;
 import org.opensearch.common.inject.Key;
 
@@ -42,8 +43,9 @@
  * @author jessewilson@google.com (Jesse Wilson)
  * @since 2.0
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface PrivateElements extends Element {
 
     /**
diff --git a/server/src/main/java/org/opensearch/common/inject/spi/ProviderBinding.java b/server/src/main/java/org/opensearch/common/inject/spi/ProviderBinding.java
index 0a63fefc0a9e9..dd55e9805843f 100644
--- a/server/src/main/java/org/opensearch/common/inject/spi/ProviderBinding.java
+++ b/server/src/main/java/org/opensearch/common/inject/spi/ProviderBinding.java
@@ -29,6 +29,7 @@
 
 package org.opensearch.common.inject.spi;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.inject.Binding;
 import org.opensearch.common.inject.Key;
 import org.opensearch.common.inject.Provider;
@@ -40,8 +41,9 @@
  * @author jessewilson@google.com (Jesse Wilson)
  * @since 2.0
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface ProviderBinding<T extends Provider<?>> extends Binding<T> {
 
     /**
diff --git a/server/src/main/java/org/opensearch/common/inject/spi/ProviderInstanceBinding.java b/server/src/main/java/org/opensearch/common/inject/spi/ProviderInstanceBinding.java
index 654f40e627e4b..25bac3b5df34c 100644
--- a/server/src/main/java/org/opensearch/common/inject/spi/ProviderInstanceBinding.java
+++ b/server/src/main/java/org/opensearch/common/inject/spi/ProviderInstanceBinding.java
@@ -29,6 +29,7 @@
 
 package org.opensearch.common.inject.spi;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.inject.Binding;
 import org.opensearch.common.inject.Provider;
 
@@ -41,8 +42,9 @@
  * @author jessewilson@google.com (Jesse Wilson)
  * @since 2.0
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface ProviderInstanceBinding<T> extends Binding<T>, HasDependencies {
 
     /**
diff --git a/server/src/main/java/org/opensearch/common/inject/spi/ProviderKeyBinding.java b/server/src/main/java/org/opensearch/common/inject/spi/ProviderKeyBinding.java
index 6f1ae8f2b9a03..f68e1662ad124 100644
--- a/server/src/main/java/org/opensearch/common/inject/spi/ProviderKeyBinding.java
+++ b/server/src/main/java/org/opensearch/common/inject/spi/ProviderKeyBinding.java
@@ -29,6 +29,7 @@
 
 package org.opensearch.common.inject.spi;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.inject.Binding;
 import org.opensearch.common.inject.Key;
 import org.opensearch.common.inject.Provider;
@@ -40,8 +41,9 @@
  * @author jessewilson@google.com (Jesse Wilson)
  * @since 2.0
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface ProviderKeyBinding<T> extends Binding<T> {
 
     /**
diff --git a/server/src/main/java/org/opensearch/common/inject/spi/ProviderLookup.java b/server/src/main/java/org/opensearch/common/inject/spi/ProviderLookup.java
index 16060ddd3e222..6afe7346a1431 100644
--- a/server/src/main/java/org/opensearch/common/inject/spi/ProviderLookup.java
+++ b/server/src/main/java/org/opensearch/common/inject/spi/ProviderLookup.java
@@ -29,6 +29,7 @@
 
 package org.opensearch.common.inject.spi;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.inject.Binder;
 import org.opensearch.common.inject.Key;
 import org.opensearch.common.inject.Provider;
@@ -45,8 +46,9 @@
  * @author jessewilson@google.com (Jesse Wilson)
  * @since 2.0
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class ProviderLookup<T> implements Element {
 
     /**
diff --git a/server/src/main/java/org/opensearch/common/inject/spi/ScopeBinding.java b/server/src/main/java/org/opensearch/common/inject/spi/ScopeBinding.java
index 7a619456e06e3..ca03f4291a062 100644
--- a/server/src/main/java/org/opensearch/common/inject/spi/ScopeBinding.java
+++ b/server/src/main/java/org/opensearch/common/inject/spi/ScopeBinding.java
@@ -29,6 +29,7 @@
 
 package org.opensearch.common.inject.spi;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.inject.Binder;
 import org.opensearch.common.inject.Scope;
 
@@ -46,8 +47,9 @@
  * @author jessewilson@google.com (Jesse Wilson)
  * @since 2.0
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class ScopeBinding implements Element {
     private final Object source;
     private final Class<? extends Annotation> annotationType;
diff --git a/server/src/main/java/org/opensearch/common/inject/spi/StaticInjectionRequest.java b/server/src/main/java/org/opensearch/common/inject/spi/StaticInjectionRequest.java
index 494e35e6c4490..c426639d85cab 100644
--- a/server/src/main/java/org/opensearch/common/inject/spi/StaticInjectionRequest.java
+++ b/server/src/main/java/org/opensearch/common/inject/spi/StaticInjectionRequest.java
@@ -29,6 +29,7 @@
 
 package org.opensearch.common.inject.spi;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.inject.Binder;
 import org.opensearch.common.inject.ConfigurationException;
 
@@ -45,8 +46,9 @@
  * @author jessewilson@google.com (Jesse Wilson)
  * @since 2.0
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class StaticInjectionRequest implements Element {
     private final Object source;
     private final Class<?> type;
diff --git a/server/src/main/java/org/opensearch/common/inject/spi/TypeConverter.java b/server/src/main/java/org/opensearch/common/inject/spi/TypeConverter.java
index 93a0f607ddc27..2386c1e528db6 100644
--- a/server/src/main/java/org/opensearch/common/inject/spi/TypeConverter.java
+++ b/server/src/main/java/org/opensearch/common/inject/spi/TypeConverter.java
@@ -29,6 +29,7 @@
 
 package org.opensearch.common.inject.spi;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.inject.TypeLiteral;
 
 /**
@@ -37,8 +38,9 @@
  * @author crazybob@google.com (Bob Lee)
  * @since 2.0
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface TypeConverter {
 
     /**
diff --git a/server/src/main/java/org/opensearch/common/inject/spi/TypeConverterBinding.java b/server/src/main/java/org/opensearch/common/inject/spi/TypeConverterBinding.java
index 00b8c7c013b5a..59311de0fb3f5 100644
--- a/server/src/main/java/org/opensearch/common/inject/spi/TypeConverterBinding.java
+++ b/server/src/main/java/org/opensearch/common/inject/spi/TypeConverterBinding.java
@@ -29,6 +29,7 @@
 
 package org.opensearch.common.inject.spi;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.inject.Binder;
 import org.opensearch.common.inject.TypeLiteral;
 import org.opensearch.common.inject.matcher.Matcher;
@@ -45,8 +46,9 @@
  * @author jessewilson@google.com (Jesse Wilson)
  * @since 2.0
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class TypeConverterBinding implements Element {
     private final Object source;
     private final Matcher<? super TypeLiteral<?>> typeMatcher;
diff --git a/server/src/main/java/org/opensearch/common/inject/spi/TypeEncounter.java b/server/src/main/java/org/opensearch/common/inject/spi/TypeEncounter.java
index e06751668c0f1..61756a5bcad95 100644
--- a/server/src/main/java/org/opensearch/common/inject/spi/TypeEncounter.java
+++ b/server/src/main/java/org/opensearch/common/inject/spi/TypeEncounter.java
@@ -29,6 +29,7 @@
 
 package org.opensearch.common.inject.spi;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.inject.Key;
 import org.opensearch.common.inject.MembersInjector;
 import org.opensearch.common.inject.Provider;
@@ -43,8 +44,9 @@
  * @param <I> the injectable type encountered
  * @since 2.0
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 @SuppressWarnings("overloads")
 public interface TypeEncounter<I> {
 
diff --git a/server/src/main/java/org/opensearch/common/inject/spi/TypeListener.java b/server/src/main/java/org/opensearch/common/inject/spi/TypeListener.java
index fd7004aa80df0..3157fa15f471b 100644
--- a/server/src/main/java/org/opensearch/common/inject/spi/TypeListener.java
+++ b/server/src/main/java/org/opensearch/common/inject/spi/TypeListener.java
@@ -29,6 +29,7 @@
 
 package org.opensearch.common.inject.spi;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.inject.TypeLiteral;
 
 /**
@@ -43,8 +44,9 @@
  *
  * @since 2.0
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface TypeListener {
 
     /**
diff --git a/server/src/main/java/org/opensearch/common/inject/spi/TypeListenerBinding.java b/server/src/main/java/org/opensearch/common/inject/spi/TypeListenerBinding.java
index 505028f09232d..4ddcf3fc11bc1 100644
--- a/server/src/main/java/org/opensearch/common/inject/spi/TypeListenerBinding.java
+++ b/server/src/main/java/org/opensearch/common/inject/spi/TypeListenerBinding.java
@@ -29,6 +29,7 @@
 
 package org.opensearch.common.inject.spi;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.inject.Binder;
 import org.opensearch.common.inject.TypeLiteral;
 import org.opensearch.common.inject.matcher.Matcher;
@@ -42,8 +43,9 @@
  * @author jessewilson@google.com (Jesse Wilson)
  * @since 2.0
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class TypeListenerBinding implements Element {
 
     private final Object source;
diff --git a/server/src/main/java/org/opensearch/common/inject/spi/UntargettedBinding.java b/server/src/main/java/org/opensearch/common/inject/spi/UntargettedBinding.java
index 37e40d45cb5a9..56890efdfcd8d 100644
--- a/server/src/main/java/org/opensearch/common/inject/spi/UntargettedBinding.java
+++ b/server/src/main/java/org/opensearch/common/inject/spi/UntargettedBinding.java
@@ -29,6 +29,7 @@
 
 package org.opensearch.common.inject.spi;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.inject.Binding;
 
 /**
@@ -38,6 +39,7 @@
  * @author jessewilson@google.com (Jesse Wilson)
  * @since 2.0
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface UntargettedBinding<T> extends Binding<T> {}
diff --git a/server/src/main/java/org/opensearch/common/lucene/index/OpenSearchDirectoryReader.java b/server/src/main/java/org/opensearch/common/lucene/index/OpenSearchDirectoryReader.java
index f9a87b9e74214..ec2cfde84ca5f 100644
--- a/server/src/main/java/org/opensearch/common/lucene/index/OpenSearchDirectoryReader.java
+++ b/server/src/main/java/org/opensearch/common/lucene/index/OpenSearchDirectoryReader.java
@@ -84,8 +84,10 @@ public DelegatingCacheHelper getDelegatingCacheHelper() {
 
     /**
      * Wraps existing IndexReader cache helper which internally provides a way to wrap CacheKey.
-     * @opensearch.internal
+     *
+     * @opensearch.api
      */
+    @PublicApi(since = "2.13.0")
     public class DelegatingCacheHelper implements CacheHelper {
         private final CacheHelper cacheHelper;
         private final DelegatingCacheKey serializableCacheKey;
@@ -113,7 +115,10 @@ public void addClosedListener(ClosedListener listener) {
     /**
      *  Wraps internal IndexReader.CacheKey and attaches a uniqueId to it which can be eventually be used instead of
      *  object itself for serialization purposes.
+     *
+     *  @opensearch.api
      */
+    @PublicApi(since = "2.13.0")
     public class DelegatingCacheKey {
         private final CacheKey cacheKey;
         private final String uniqueId;
diff --git a/server/src/main/java/org/opensearch/common/metrics/MeanMetric.java b/server/src/main/java/org/opensearch/common/metrics/MeanMetric.java
index 359facdce633b..94d44d5b35d74 100644
--- a/server/src/main/java/org/opensearch/common/metrics/MeanMetric.java
+++ b/server/src/main/java/org/opensearch/common/metrics/MeanMetric.java
@@ -32,13 +32,16 @@
 
 package org.opensearch.common.metrics;
 
+import org.opensearch.common.annotation.PublicApi;
+
 import java.util.concurrent.atomic.LongAdder;
 
 /**
  * An average metric for tracking.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class MeanMetric implements Metric {
 
     private final LongAdder counter = new LongAdder();
diff --git a/server/src/main/java/org/opensearch/common/metrics/OperationMetrics.java b/server/src/main/java/org/opensearch/common/metrics/OperationMetrics.java
index 97fbbc2ce5cde..71c4a29f0f610 100644
--- a/server/src/main/java/org/opensearch/common/metrics/OperationMetrics.java
+++ b/server/src/main/java/org/opensearch/common/metrics/OperationMetrics.java
@@ -37,6 +37,14 @@ public void before() {
         current.incrementAndGet();
     }
 
+    /**
+     * Invoke before the given operation begins in multiple items at the same time.
+     * @param n number of items
+     */
+    public void beforeN(int n) {
+        current.addAndGet(n);
+    }
+
     /**
      * Invoked upon completion (success or failure) of the given operation
      * @param currentTime elapsed time of the operation
@@ -46,6 +54,18 @@ public void after(long currentTime) {
         time.inc(currentTime);
     }
 
+    /**
+     * Invoked upon completion (success or failure) of the given operation for multiple items.
+     * @param n number of items completed
+     * @param currentTime elapsed time of the operation
+     */
+    public void afterN(int n, long currentTime) {
+        current.addAndGet(-n);
+        for (int i = 0; i < n; ++i) {
+            time.inc(currentTime);
+        }
+    }
+
     /**
      * Invoked upon failure of the operation.
      */
@@ -53,6 +73,16 @@ public void failed() {
         failed.inc();
     }
 
+    /**
+     * Invoked upon failure of the operation on multiple items.
+     * @param n number of items on operation.
+     */
+    public void failedN(int n) {
+        for (int i = 0; i < n; ++i) {
+            failed.inc();
+        }
+    }
+
     public void add(OperationMetrics other) {
         // Don't try copying over current, since in-flight requests will be linked to the existing metrics instance.
         failed.inc(other.failed.count());
diff --git a/server/src/main/java/org/opensearch/common/network/NetworkModule.java b/server/src/main/java/org/opensearch/common/network/NetworkModule.java
index d0f5dd9e4581d..bb8da190a6f35 100644
--- a/server/src/main/java/org/opensearch/common/network/NetworkModule.java
+++ b/server/src/main/java/org/opensearch/common/network/NetworkModule.java
@@ -55,6 +55,8 @@
 import org.opensearch.http.HttpServerTransport;
 import org.opensearch.index.shard.PrimaryReplicaSyncer.ResyncTask;
 import org.opensearch.plugins.NetworkPlugin;
+import org.opensearch.plugins.SecureHttpTransportSettingsProvider;
+import org.opensearch.plugins.SecureSettingsFactory;
 import org.opensearch.plugins.SecureTransportSettingsProvider;
 import org.opensearch.ratelimitting.admissioncontrol.enums.AdmissionControlActionType;
 import org.opensearch.tasks.RawTaskStatus;
@@ -74,7 +76,9 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.function.Supplier;
+import java.util.stream.Collectors;
 
 /**
  * A module to handle registering and binding all network related classes.
@@ -173,13 +177,31 @@ public NetworkModule(
         ClusterSettings clusterSettings,
         Tracer tracer,
         List<TransportInterceptor> transportInterceptors,
-        Collection<SecureTransportSettingsProvider> secureTransportSettingsProvider
+        Collection<SecureSettingsFactory> secureSettingsFactories
     ) {
         this.settings = settings;
 
-        if (secureTransportSettingsProvider.size() > 1) {
+        final Collection<SecureTransportSettingsProvider> secureTransportSettingsProviders = secureSettingsFactories.stream()
+            .map(p -> p.getSecureTransportSettingsProvider(settings))
+            .filter(Optional::isPresent)
+            .map(Optional::get)
+            .collect(Collectors.toList());
+
+        if (secureTransportSettingsProviders.size() > 1) {
+            throw new IllegalArgumentException(
+                "there is more than one secure transport settings provider: " + secureTransportSettingsProviders
+            );
+        }
+
+        final Collection<SecureHttpTransportSettingsProvider> secureHttpTransportSettingsProviders = secureSettingsFactories.stream()
+            .map(p -> p.getSecureHttpTransportSettingsProvider(settings))
+            .filter(Optional::isPresent)
+            .map(Optional::get)
+            .collect(Collectors.toList());
+
+        if (secureHttpTransportSettingsProviders.size() > 1) {
             throw new IllegalArgumentException(
-                "there is more than one secure transport settings provider: " + secureTransportSettingsProvider
+                "there is more than one secure HTTP transport settings provider: " + secureHttpTransportSettingsProviders
             );
         }
 
@@ -213,9 +235,9 @@ public NetworkModule(
                 registerTransport(entry.getKey(), entry.getValue());
             }
 
-            // Register any secure transports if available
-            if (secureTransportSettingsProvider.isEmpty() == false) {
-                final SecureTransportSettingsProvider secureSettingProvider = secureTransportSettingsProvider.iterator().next();
+            // Register any HTTP secure transports if available
+            if (secureHttpTransportSettingsProviders.isEmpty() == false) {
+                final SecureHttpTransportSettingsProvider secureSettingProvider = secureHttpTransportSettingsProviders.iterator().next();
 
                 final Map<String, Supplier<HttpServerTransport>> secureHttpTransportFactory = plugin.getSecureHttpTransports(
                     settings,
@@ -233,6 +255,11 @@ public NetworkModule(
                 for (Map.Entry<String, Supplier<HttpServerTransport>> entry : secureHttpTransportFactory.entrySet()) {
                     registerHttpTransport(entry.getKey(), entry.getValue());
                 }
+            }
+
+            // Register any secure transports if available
+            if (secureTransportSettingsProviders.isEmpty() == false) {
+                final SecureTransportSettingsProvider secureSettingProvider = secureTransportSettingsProviders.iterator().next();
 
                 final Map<String, Supplier<Transport>> secureTransportFactory = plugin.getSecureTransports(
                     settings,
diff --git a/server/src/main/java/org/opensearch/common/regex/Regex.java b/server/src/main/java/org/opensearch/common/regex/Regex.java
index 323b460af62df..6d8b5c3585c4c 100644
--- a/server/src/main/java/org/opensearch/common/regex/Regex.java
+++ b/server/src/main/java/org/opensearch/common/regex/Regex.java
@@ -35,6 +35,7 @@
 import org.apache.lucene.util.automaton.Automata;
 import org.apache.lucene.util.automaton.Automaton;
 import org.apache.lucene.util.automaton.Operations;
+import org.opensearch.common.Glob;
 import org.opensearch.core.common.Strings;
 
 import java.util.ArrayList;
@@ -125,39 +126,7 @@ public static boolean simpleMatch(String pattern, String str, boolean caseInsens
             pattern = Strings.toLowercaseAscii(pattern);
             str = Strings.toLowercaseAscii(str);
         }
-        return simpleMatchWithNormalizedStrings(pattern, str);
-    }
-
-    private static boolean simpleMatchWithNormalizedStrings(String pattern, String str) {
-        int sIdx = 0, pIdx = 0, match = 0, wildcardIdx = -1;
-        while (sIdx < str.length()) {
-            // both chars matching, incrementing both pointers
-            if (pIdx < pattern.length() && str.charAt(sIdx) == pattern.charAt(pIdx)) {
-                sIdx++;
-                pIdx++;
-            } else if (pIdx < pattern.length() && pattern.charAt(pIdx) == '*') {
-                // wildcard found, only incrementing pattern pointer
-                wildcardIdx = pIdx;
-                match = sIdx;
-                pIdx++;
-            } else if (wildcardIdx != -1) {
-                // last pattern pointer was a wildcard, incrementing string pointer
-                pIdx = wildcardIdx + 1;
-                match++;
-                sIdx = match;
-            } else {
-                // current pattern pointer is not a wildcard, last pattern pointer was also not a wildcard
-                // characters do not match
-                return false;
-            }
-        }
-
-        // check for remaining characters in pattern
-        while (pIdx < pattern.length() && pattern.charAt(pIdx) == '*') {
-            pIdx++;
-        }
-
-        return pIdx == pattern.length();
+        return Glob.globMatch(pattern, str);
     }
 
     /**
diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
index 4f1815de224db..4a5a45eb1a17a 100644
--- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
+++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java
@@ -64,6 +64,7 @@
 import org.opensearch.cluster.routing.OperationRouting;
 import org.opensearch.cluster.routing.allocation.AwarenessReplicaBalance;
 import org.opensearch.cluster.routing.allocation.DiskThresholdSettings;
+import org.opensearch.cluster.routing.allocation.ExistingShardsAllocator;
 import org.opensearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
 import org.opensearch.cluster.routing.allocation.decider.AwarenessAllocationDecider;
 import org.opensearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider;
@@ -83,6 +84,7 @@
 import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.cache.CacheType;
 import org.opensearch.common.cache.settings.CacheSettings;
+import org.opensearch.common.cache.store.settings.OpenSearchOnHeapCacheSettings;
 import org.opensearch.common.logging.Loggers;
 import org.opensearch.common.network.NetworkModule;
 import org.opensearch.common.network.NetworkService;
@@ -101,6 +103,7 @@
 import org.opensearch.gateway.DanglingIndicesState;
 import org.opensearch.gateway.GatewayService;
 import org.opensearch.gateway.PersistedClusterStateService;
+import org.opensearch.gateway.ShardsBatchGatewayAllocator;
 import org.opensearch.gateway.remote.RemoteClusterStateService;
 import org.opensearch.http.HttpTransportSettings;
 import org.opensearch.index.IndexModule;
@@ -117,6 +120,7 @@
 import org.opensearch.indices.IndicesQueryCache;
 import org.opensearch.indices.IndicesRequestCache;
 import org.opensearch.indices.IndicesService;
+import org.opensearch.indices.RemoteStoreSettings;
 import org.opensearch.indices.ShardLimitValidator;
 import org.opensearch.indices.analysis.HunspellService;
 import org.opensearch.indices.breaker.BreakerSettings;
@@ -251,7 +255,9 @@ public void apply(Settings value, Settings current, Settings previous) {
                 AwarenessReplicaBalance.CLUSTER_ROUTING_ALLOCATION_AWARENESS_BALANCE_SETTING,
                 BalancedShardsAllocator.INDEX_BALANCE_FACTOR_SETTING,
                 BalancedShardsAllocator.SHARD_BALANCE_FACTOR_SETTING,
+                BalancedShardsAllocator.PRIMARY_SHARD_REBALANCE_BUFFER,
                 BalancedShardsAllocator.PREFER_PRIMARY_SHARD_BALANCE,
+                BalancedShardsAllocator.PREFER_PRIMARY_SHARD_REBALANCE,
                 BalancedShardsAllocator.SHARD_MOVE_PRIMARY_FIRST_SETTING,
                 BalancedShardsAllocator.SHARD_MOVEMENT_STRATEGY_SETTING,
                 BalancedShardsAllocator.THRESHOLD_SETTING,
@@ -264,6 +270,7 @@ public void apply(Settings value, Settings current, Settings previous) {
                 DanglingIndicesState.AUTO_IMPORT_DANGLING_INDICES_SETTING,
                 EnableAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ENABLE_SETTING,
                 EnableAllocationDecider.CLUSTER_ROUTING_REBALANCE_ENABLE_SETTING,
+                ExistingShardsAllocator.EXISTING_SHARDS_ALLOCATOR_BATCH_MODE,
                 FilterAllocationDecider.CLUSTER_ROUTING_INCLUDE_GROUP_SETTING,
                 FilterAllocationDecider.CLUSTER_ROUTING_EXCLUDE_GROUP_SETTING,
                 FilterAllocationDecider.CLUSTER_ROUTING_REQUIRE_GROUP_SETTING,
@@ -288,6 +295,7 @@ public void apply(Settings value, Settings current, Settings previous) {
                 ShardLimitValidator.SETTING_CLUSTER_MAX_SHARDS_PER_CLUSTER,
                 ShardLimitValidator.SETTING_CLUSTER_IGNORE_DOT_INDEXES,
                 RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING,
+                RecoverySettings.INDICES_REPLICATION_MAX_BYTES_PER_SEC_SETTING,
                 RecoverySettings.INDICES_RECOVERY_RETRY_DELAY_STATE_SYNC_SETTING,
                 RecoverySettings.INDICES_RECOVERY_RETRY_DELAY_NETWORK_SETTING,
                 RecoverySettings.INDICES_RECOVERY_ACTIVITY_TIMEOUT_SETTING,
@@ -297,7 +305,6 @@ public void apply(Settings value, Settings current, Settings previous) {
                 RecoverySettings.INDICES_RECOVERY_MAX_CONCURRENT_OPERATIONS_SETTING,
                 RecoverySettings.INDICES_RECOVERY_MAX_CONCURRENT_REMOTE_STORE_STREAMS_SETTING,
                 RecoverySettings.INDICES_INTERNAL_REMOTE_UPLOAD_TIMEOUT,
-                RecoverySettings.CLUSTER_REMOTE_INDEX_SEGMENT_METADATA_RETENTION_MAX_COUNT_SETTING,
                 ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_INITIAL_PRIMARIES_RECOVERIES_SETTING,
                 ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_INITIAL_REPLICAS_RECOVERIES_SETTING,
                 ThrottlingAllocationDecider.CLUSTER_ROUTING_ALLOCATION_NODE_CONCURRENT_INCOMING_RECOVERIES_SETTING,
@@ -326,6 +333,7 @@ public void apply(Settings value, Settings current, Settings previous) {
                 GatewayService.RECOVER_AFTER_MASTER_NODES_SETTING,
                 GatewayService.RECOVER_AFTER_NODES_SETTING,
                 GatewayService.RECOVER_AFTER_TIME_SETTING,
+                ShardsBatchGatewayAllocator.GATEWAY_ALLOCATOR_BATCH_SIZE,
                 PersistedClusterStateService.SLOW_WRITE_LOGGING_THRESHOLD,
                 NetworkModule.HTTP_DEFAULT_TYPE_SETTING,
                 NetworkModule.TRANSPORT_DEFAULT_TYPE_SETTING,
@@ -489,6 +497,8 @@ public void apply(Settings value, Settings current, Settings previous) {
                 IndicesFieldDataCache.INDICES_FIELDDATA_CACHE_SIZE_KEY,
                 IndicesRequestCache.INDICES_CACHE_QUERY_SIZE,
                 IndicesRequestCache.INDICES_CACHE_QUERY_EXPIRE,
+                IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_INTERVAL_SETTING,
+                IndicesRequestCache.INDICES_REQUEST_CACHE_STALENESS_THRESHOLD_SETTING,
                 HunspellService.HUNSPELL_LAZY_LOAD,
                 HunspellService.HUNSPELL_IGNORE_CASE,
                 HunspellService.HUNSPELL_DICTIONARY_OPTIONS,
@@ -521,6 +531,7 @@ public void apply(Settings value, Settings current, Settings previous) {
                 SearchService.MAX_OPEN_SCROLL_CONTEXT,
                 SearchService.MAX_OPEN_PIT_CONTEXT,
                 SearchService.MAX_PIT_KEEPALIVE_SETTING,
+                SearchService.MAX_AGGREGATION_REWRITE_FILTERS,
                 CreatePitController.PIT_INIT_KEEP_ALIVE,
                 Node.WRITE_PORTS_FILE_SETTING,
                 Node.NODE_NAME_SETTING,
@@ -706,10 +717,8 @@ public void apply(Settings value, Settings current, Settings previous) {
                 RemoteClusterStateService.METADATA_MANIFEST_UPLOAD_TIMEOUT_SETTING,
                 RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING,
                 RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING,
-                IndicesService.CLUSTER_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING,
                 IndicesService.CLUSTER_REMOTE_INDEX_RESTRICT_ASYNC_DURABILITY_SETTING,
                 IndicesService.CLUSTER_INDEX_RESTRICT_REPLICATION_TYPE_SETTING,
-                IndicesService.CLUSTER_REMOTE_STORE_PATH_PREFIX_TYPE_SETTING,
 
                 // Admission Control Settings
                 AdmissionControlSettings.ADMISSION_CONTROL_TRANSPORT_LAYER_MODE,
@@ -723,7 +732,14 @@ public void apply(Settings value, Settings current, Settings previous) {
 
                 // Concurrent segment search settings
                 SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING,
-                SearchService.CONCURRENT_SEGMENT_SEARCH_TARGET_MAX_SLICE_COUNT_SETTING
+                SearchService.CONCURRENT_SEGMENT_SEARCH_TARGET_MAX_SLICE_COUNT_SETTING,
+
+                RemoteStoreSettings.CLUSTER_REMOTE_INDEX_SEGMENT_METADATA_RETENTION_MAX_COUNT_SETTING,
+                RemoteStoreSettings.CLUSTER_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING,
+                RemoteStoreSettings.CLUSTER_REMOTE_TRANSLOG_TRANSFER_TIMEOUT_SETTING,
+                RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING,
+                RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING,
+                RemoteStoreSettings.CLUSTER_REMOTE_MAX_TRANSLOG_READERS
             )
         )
     );
@@ -745,6 +761,14 @@ public void apply(Settings value, Settings current, Settings previous) {
             TelemetrySettings.METRICS_FEATURE_ENABLED_SETTING
         ),
         List.of(FeatureFlags.PLUGGABLE_CACHE),
-        List.of(CacheSettings.getConcreteStoreNameSettingForCacheType(CacheType.INDICES_REQUEST_CACHE))
+        List.of(
+            CacheSettings.getConcreteStoreNameSettingForCacheType(CacheType.INDICES_REQUEST_CACHE),
+            OpenSearchOnHeapCacheSettings.MAXIMUM_SIZE_IN_BYTES.getConcreteSettingForNamespace(
+                CacheType.INDICES_REQUEST_CACHE.getSettingPrefix()
+            ),
+            OpenSearchOnHeapCacheSettings.EXPIRE_AFTER_ACCESS_SETTING.getConcreteSettingForNamespace(
+                CacheType.INDICES_REQUEST_CACHE.getSettingPrefix()
+            )
+        )
     );
 }
diff --git a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java
index c6c312d6b6eea..980c432774f6e 100644
--- a/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java
+++ b/server/src/main/java/org/opensearch/common/settings/IndexScopedSettings.java
@@ -198,6 +198,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
                 EngineConfig.INDEX_CODEC_SETTING,
                 EngineConfig.INDEX_CODEC_COMPRESSION_LEVEL_SETTING,
                 EngineConfig.INDEX_OPTIMIZE_AUTO_GENERATED_IDS,
+                EngineConfig.INDEX_USE_COMPOUND_FILE,
                 IndexMetadata.SETTING_WAIT_FOR_ACTIVE_SHARDS,
                 IndexSettings.DEFAULT_PIPELINE,
                 IndexSettings.FINAL_PIPELINE,
diff --git a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java
index 8633cf1fe25ea..a72583607ede0 100644
--- a/server/src/main/java/org/opensearch/common/util/FeatureFlags.java
+++ b/server/src/main/java/org/opensearch/common/util/FeatureFlags.java
@@ -12,9 +12,11 @@
 import org.opensearch.common.settings.Setting.Property;
 import org.opensearch.common.settings.Settings;
 
+import java.util.List;
+
 /**
  * Utility class to manage feature flags. Feature flags are system properties that must be set on the JVM.
- * These are used to gate the visibility/availability of incomplete features. Fore more information, see
+ * These are used to gate the visibility/availability of incomplete features. For more information, see
  * https://featureflags.io/feature-flag-introduction/
  *
  * @opensearch.internal
@@ -65,11 +67,54 @@ public class FeatureFlags {
      */
     public static final String PLUGGABLE_CACHE = "opensearch.experimental.feature.pluggable.caching.enabled";
 
+    public static final Setting<Boolean> REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING = Setting.boolSetting(
+        REMOTE_STORE_MIGRATION_EXPERIMENTAL,
+        false,
+        Property.NodeScope
+    );
+
+    public static final Setting<Boolean> EXTENSIONS_SETTING = Setting.boolSetting(EXTENSIONS, false, Property.NodeScope);
+
+    public static final Setting<Boolean> IDENTITY_SETTING = Setting.boolSetting(IDENTITY, false, Property.NodeScope);
+
+    public static final Setting<Boolean> TELEMETRY_SETTING = Setting.boolSetting(TELEMETRY, false, Property.NodeScope);
+
+    public static final Setting<Boolean> DATETIME_FORMATTER_CACHING_SETTING = Setting.boolSetting(
+        DATETIME_FORMATTER_CACHING,
+        false,
+        Property.NodeScope
+    );
+
+    public static final Setting<Boolean> WRITEABLE_REMOTE_INDEX_SETTING = Setting.boolSetting(
+        WRITEABLE_REMOTE_INDEX,
+        false,
+        Property.NodeScope
+    );
+
+    public static final Setting<Boolean> PLUGGABLE_CACHE_SETTING = Setting.boolSetting(PLUGGABLE_CACHE, false, Property.NodeScope);
+
+    private static final List<Setting<Boolean>> ALL_FEATURE_FLAG_SETTINGS = List.of(
+        REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING,
+        EXTENSIONS_SETTING,
+        IDENTITY_SETTING,
+        TELEMETRY_SETTING,
+        DATETIME_FORMATTER_CACHING_SETTING,
+        WRITEABLE_REMOTE_INDEX_SETTING,
+        PLUGGABLE_CACHE_SETTING
+    );
     /**
      * Should store the settings from opensearch.yml.
      */
     private static Settings settings;
 
+    static {
+        Settings.Builder settingsBuilder = Settings.builder();
+        for (Setting<Boolean> ffSetting : ALL_FEATURE_FLAG_SETTINGS) {
+            settingsBuilder = settingsBuilder.put(ffSetting.getKey(), ffSetting.getDefault(Settings.EMPTY));
+        }
+        settings = settingsBuilder.build();
+    }
+
     /**
      * This method is responsible to map settings from opensearch.yml to local stored
      * settings value. That is used for the existing isEnabled method.
@@ -77,7 +122,14 @@ public class FeatureFlags {
      * @param openSearchSettings The settings stored in opensearch.yml.
      */
     public static void initializeFeatureFlags(Settings openSearchSettings) {
-        settings = openSearchSettings;
+        Settings.Builder settingsBuilder = Settings.builder();
+        for (Setting<Boolean> ffSetting : ALL_FEATURE_FLAG_SETTINGS) {
+            settingsBuilder = settingsBuilder.put(
+                ffSetting.getKey(),
+                openSearchSettings.getAsBoolean(ffSetting.getKey(), ffSetting.getDefault(openSearchSettings))
+            );
+        }
+        settings = settingsBuilder.build();
     }
 
     /**
@@ -103,30 +155,4 @@ public static boolean isEnabled(Setting<Boolean> featureFlag) {
             return featureFlag.getDefault(Settings.EMPTY);
         }
     }
-
-    public static final Setting<Boolean> REMOTE_STORE_MIGRATION_EXPERIMENTAL_SETTING = Setting.boolSetting(
-        REMOTE_STORE_MIGRATION_EXPERIMENTAL,
-        false,
-        Property.NodeScope
-    );
-
-    public static final Setting<Boolean> EXTENSIONS_SETTING = Setting.boolSetting(EXTENSIONS, false, Property.NodeScope);
-
-    public static final Setting<Boolean> IDENTITY_SETTING = Setting.boolSetting(IDENTITY, false, Property.NodeScope);
-
-    public static final Setting<Boolean> TELEMETRY_SETTING = Setting.boolSetting(TELEMETRY, false, Property.NodeScope);
-
-    public static final Setting<Boolean> DATETIME_FORMATTER_CACHING_SETTING = Setting.boolSetting(
-        DATETIME_FORMATTER_CACHING,
-        true,
-        Property.NodeScope
-    );
-
-    public static final Setting<Boolean> WRITEABLE_REMOTE_INDEX_SETTING = Setting.boolSetting(
-        WRITEABLE_REMOTE_INDEX,
-        false,
-        Property.NodeScope
-    );
-
-    public static final Setting<Boolean> PLUGGABLE_CACHE_SETTING = Setting.boolSetting(PLUGGABLE_CACHE, false, Property.NodeScope);
 }
diff --git a/server/src/main/java/org/opensearch/common/util/IndexUtils.java b/server/src/main/java/org/opensearch/common/util/IndexUtils.java
new file mode 100644
index 0000000000000..b4f049617c4fc
--- /dev/null
+++ b/server/src/main/java/org/opensearch/common/util/IndexUtils.java
@@ -0,0 +1,128 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.util;
+
+import org.opensearch.action.support.IndicesOptions;
+import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
+import org.opensearch.common.regex.Regex;
+import org.opensearch.index.IndexNotFoundException;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * Common Utility methods for Indices.
+ *
+ * @opensearch.internal
+ */
+public class IndexUtils {
+
+    /**
+     * Filters out list of available indices based on the list of selected indices.
+     *
+     * @param availableIndices list of available indices
+     * @param selectedIndices  list of selected indices
+     * @param indicesOptions    ignore indices flag
+     * @return filtered out indices
+     */
+    public static List<String> filterIndices(List<String> availableIndices, String[] selectedIndices, IndicesOptions indicesOptions) {
+        if (IndexNameExpressionResolver.isAllIndices(Arrays.asList(selectedIndices))) {
+            return availableIndices;
+        }
+
+        // Move the exclusions to end of list to ensure they are processed
+        // after explicitly selected indices are chosen.
+        final List<String> excludesAtEndSelectedIndices = Stream.concat(
+            Arrays.stream(selectedIndices).filter(s -> s.isEmpty() || s.charAt(0) != '-'),
+            Arrays.stream(selectedIndices).filter(s -> !s.isEmpty() && s.charAt(0) == '-')
+        ).collect(Collectors.toUnmodifiableList());
+
+        Set<String> result = null;
+        for (int i = 0; i < excludesAtEndSelectedIndices.size(); i++) {
+            String indexOrPattern = excludesAtEndSelectedIndices.get(i);
+            boolean add = true;
+            if (!indexOrPattern.isEmpty()) {
+                if (availableIndices.contains(indexOrPattern)) {
+                    if (result == null) {
+                        result = new HashSet<>();
+                    }
+                    result.add(indexOrPattern);
+                    continue;
+                }
+                if (indexOrPattern.charAt(0) == '+') {
+                    add = true;
+                    indexOrPattern = indexOrPattern.substring(1);
+                    // if its the first, add empty set
+                    if (i == 0) {
+                        result = new HashSet<>();
+                    }
+                } else if (indexOrPattern.charAt(0) == '-') {
+                    // If the first index pattern is an exclusion, then all patterns are exclusions due to the
+                    // reordering logic above. In this case, the request is interpreted as "include all indexes except
+                    // those matching the exclusions" so we add all indices here and then remove the ones that match the exclusion patterns.
+                    if (i == 0) {
+                        result = new HashSet<>(availableIndices);
+                    }
+                    add = false;
+                    indexOrPattern = indexOrPattern.substring(1);
+                }
+            }
+            if (indexOrPattern.isEmpty() || !Regex.isSimpleMatchPattern(indexOrPattern)) {
+                if (!availableIndices.contains(indexOrPattern)) {
+                    if (!indicesOptions.ignoreUnavailable()) {
+                        throw new IndexNotFoundException(indexOrPattern);
+                    } else {
+                        if (result == null) {
+                            // add all the previous ones...
+                            result = new HashSet<>(availableIndices.subList(0, i));
+                        }
+                    }
+                } else {
+                    if (result != null) {
+                        if (add) {
+                            result.add(indexOrPattern);
+                        } else {
+                            result.remove(indexOrPattern);
+                        }
+                    }
+                }
+                continue;
+            }
+            if (result == null) {
+                // add all the previous ones...
+                result = new HashSet<>(availableIndices.subList(0, i));
+            }
+            boolean found = false;
+            for (String index : availableIndices) {
+                if (Regex.simpleMatch(indexOrPattern, index)) {
+                    found = true;
+                    if (add) {
+                        result.add(index);
+                    } else {
+                        result.remove(index);
+                    }
+                }
+            }
+            if (!found && !indicesOptions.allowNoIndices()) {
+                throw new IndexNotFoundException(indexOrPattern);
+            }
+        }
+        if (result == null) {
+            return Collections.unmodifiableList(new ArrayList<>(Arrays.asList(selectedIndices)));
+        }
+        return Collections.unmodifiableList(new ArrayList<>(result));
+    }
+
+}
diff --git a/server/src/main/java/org/opensearch/common/xcontent/JsonToStringXContentParser.java b/server/src/main/java/org/opensearch/common/xcontent/JsonToStringXContentParser.java
index 9b2bd06a88e2e..998122d9e5c43 100644
--- a/server/src/main/java/org/opensearch/common/xcontent/JsonToStringXContentParser.java
+++ b/server/src/main/java/org/opensearch/common/xcontent/JsonToStringXContentParser.java
@@ -106,8 +106,9 @@ private void parseToken(StringBuilder path, String currentFieldName) throws IOEx
                 // skip
             } else if (this.parser.currentToken() == Token.START_OBJECT) {
                 parseToken(path, currentFieldName);
-                int dotIndex = path.lastIndexOf(DOT_SYMBOL);
-                if (dotIndex != -1) {
+                int dotIndex = path.lastIndexOf(DOT_SYMBOL, path.length());
+
+                if (dotIndex != -1 && path.length() > currentFieldName.length()) {
                     path.setLength(path.length() - currentFieldName.length() - 1);
                 }
             } else {
@@ -117,8 +118,8 @@ private void parseToken(StringBuilder path, String currentFieldName) throws IOEx
                 parseValue(parsedFields);
                 this.valueList.add(parsedFields.toString());
                 this.valueAndPathList.add(path + EQUAL_SYMBOL + parsedFields);
-                int dotIndex = path.lastIndexOf(DOT_SYMBOL);
-                if (dotIndex != -1) {
+                int dotIndex = path.lastIndexOf(DOT_SYMBOL, path.length());
+                if (dotIndex != -1 && path.length() > currentFieldName.length()) {
                     path.setLength(path.length() - currentFieldName.length() - 1);
                 }
             }
diff --git a/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java b/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java
index 41783b89ccc69..f4503ce55e6bc 100644
--- a/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java
+++ b/server/src/main/java/org/opensearch/extensions/rest/RestSendToExtensionAction.java
@@ -12,6 +12,7 @@
 import org.apache.logging.log4j.Logger;
 import org.opensearch.action.ActionModule.DynamicActionRegistry;
 import org.opensearch.client.node.NodeClient;
+import org.opensearch.common.annotation.ExperimentalApi;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.rest.RestStatus;
@@ -52,7 +53,10 @@
 
 /**
  * An action that forwards REST requests to an extension
+ *
+ * @opensearch.experimental
  */
+@ExperimentalApi
 public class RestSendToExtensionAction extends BaseRestHandler {
 
     private static final String SEND_TO_EXTENSION_ACTION = "send_to_extension_action";
diff --git a/server/src/main/java/org/opensearch/gateway/AsyncShardBatchFetch.java b/server/src/main/java/org/opensearch/gateway/AsyncShardBatchFetch.java
new file mode 100644
index 0000000000000..4f39a39cea678
--- /dev/null
+++ b/server/src/main/java/org/opensearch/gateway/AsyncShardBatchFetch.java
@@ -0,0 +1,243 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.gateway;
+
+import org.apache.logging.log4j.Logger;
+import org.opensearch.action.support.nodes.BaseNodeResponse;
+import org.opensearch.action.support.nodes.BaseNodesResponse;
+import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.common.logging.Loggers;
+import org.opensearch.core.index.shard.ShardId;
+import org.opensearch.indices.store.ShardAttributes;
+
+import java.lang.reflect.Array;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Predicate;
+
+import reactor.util.annotation.NonNull;
+
+/**
+ * Implementation of AsyncShardFetch with batching support. This class is responsible for executing the fetch
+ * part using the base class {@link AsyncShardFetch}. Other functionalities needed for a batch are only written here.
+ * This separation also takes care of the extra generic type V which is only needed for batch
+ * transport actions like {@link TransportNodesListGatewayStartedShardsBatch} and
+ * {@link org.opensearch.indices.store.TransportNodesListShardStoreMetadataBatch}.
+ *
+ * @param <T> Response type of the transport action.
+ * @param <V> Data type of shard level response.
+ *
+ * @opensearch.internal
+ */
+public abstract class AsyncShardBatchFetch<T extends BaseNodeResponse, V> extends AsyncShardFetch<T> {
+
+    @SuppressWarnings("unchecked")
+    AsyncShardBatchFetch(
+        Logger logger,
+        String type,
+        Map<ShardId, ShardAttributes> shardAttributesMap,
+        AsyncShardFetch.Lister<? extends BaseNodesResponse<T>, T> action,
+        String batchId,
+        Class<V> clazz,
+        V emptyShardResponse,
+        Predicate<V> emptyShardResponsePredicate,
+        ShardBatchResponseFactory<T, V> responseFactory
+    ) {
+        super(
+            logger,
+            type,
+            shardAttributesMap,
+            action,
+            batchId,
+            new ShardBatchCache<>(
+                logger,
+                type,
+                shardAttributesMap,
+                "BatchID=[" + batchId + "]",
+                clazz,
+                emptyShardResponse,
+                emptyShardResponsePredicate,
+                responseFactory
+            )
+        );
+    }
+
+    /**
+     * Remove a shard from the cache maintaining a full batch of shards. This is needed to clear the shard once it's
+     * assigned or failed.
+     *
+     * @param shardId shardId to be removed from the batch.
+     */
+    public synchronized void clearShard(ShardId shardId) {
+        this.shardAttributesMap.remove(shardId);
+        this.cache.deleteShard(shardId);
+    }
+
+    /**
+     * Cache implementation of transport actions returning batch of shards related data in the response.
+     * Store node level responses of transport actions like {@link TransportNodesListGatewayStartedShardsBatch} or
+     * {@link org.opensearch.indices.store.TransportNodesListShardStoreMetadataBatch} with memory efficient caching
+     * approach. This cache class is not thread safe, all of its methods are being called from
+     * {@link AsyncShardFetch} class which has synchronized blocks present to handle multiple threads.
+     *
+     * @param <T> Response type of transport action.
+     * @param <V> Data type of shard level response.
+     */
+    static class ShardBatchCache<T extends BaseNodeResponse, V> extends AsyncShardFetchCache<T> {
+        private final Map<String, NodeEntry<V>> cache;
+        private final Map<ShardId, Integer> shardIdToArray;
+        private final int batchSize;
+        private final Class<V> shardResponseClass;
+        private final ShardBatchResponseFactory<T, V> responseFactory;
+        private final V emptyResponse;
+        private final Predicate<V> emptyShardResponsePredicate;
+        private final Logger logger;
+
+        public ShardBatchCache(
+            Logger logger,
+            String type,
+            Map<ShardId, ShardAttributes> shardAttributesMap,
+            String logKey,
+            Class<V> clazz,
+            V emptyResponse,
+            Predicate<V> emptyShardResponsePredicate,
+            ShardBatchResponseFactory<T, V> responseFactory
+        ) {
+            super(Loggers.getLogger(logger, "_" + logKey), type);
+            this.batchSize = shardAttributesMap.size();
+            this.emptyShardResponsePredicate = emptyShardResponsePredicate;
+            cache = new HashMap<>();
+            shardIdToArray = new HashMap<>();
+            fillShardIdKeys(shardAttributesMap.keySet());
+            this.shardResponseClass = clazz;
+            this.emptyResponse = emptyResponse;
+            this.logger = logger;
+            this.responseFactory = responseFactory;
+        }
+
+        @Override
+        @NonNull
+        public Map<String, ? extends BaseNodeEntry> getCache() {
+            return cache;
+        }
+
+        @Override
+        public void deleteShard(ShardId shardId) {
+            if (shardIdToArray.containsKey(shardId)) {
+                Integer shardIdIndex = shardIdToArray.remove(shardId);
+                for (String nodeId : cache.keySet()) {
+                    cache.get(nodeId).clearShard(shardIdIndex);
+                }
+            }
+        }
+
+        @Override
+        public void initData(DiscoveryNode node) {
+            cache.put(node.getId(), new NodeEntry<>(node.getId(), shardResponseClass, batchSize, emptyShardResponsePredicate));
+        }
+
+        /**
+         * Put the response received from data nodes into the cache.
+         * Get shard level data from batch, then filter out if any shards received failures.
+         * After that complete storing the data at node level and mark fetching as done.
+         *
+         * @param node     node from which we got the response.
+         * @param response shard metadata coming from node.
+         */
+        @Override
+        public void putData(DiscoveryNode node, T response) {
+            NodeEntry<V> nodeEntry = cache.get(node.getId());
+            Map<ShardId, V> batchResponse = responseFactory.getShardBatchData(response);
+            nodeEntry.doneFetching(batchResponse, shardIdToArray);
+        }
+
+        @Override
+        public T getData(DiscoveryNode node) {
+            return this.responseFactory.getNewResponse(node, getBatchData(cache.get(node.getId())));
+        }
+
+        private HashMap<ShardId, V> getBatchData(NodeEntry<V> nodeEntry) {
+            V[] nodeShardEntries = nodeEntry.getData();
+            boolean[] emptyResponses = nodeEntry.getEmptyShardResponse();
+            HashMap<ShardId, V> shardData = new HashMap<>();
+            for (Map.Entry<ShardId, Integer> shardIdEntry : shardIdToArray.entrySet()) {
+                ShardId shardId = shardIdEntry.getKey();
+                Integer arrIndex = shardIdEntry.getValue();
+                if (emptyResponses[arrIndex]) {
+                    shardData.put(shardId, emptyResponse);
+                } else if (nodeShardEntries[arrIndex] != null) {
+                    // ignore null responses here
+                    shardData.put(shardId, nodeShardEntries[arrIndex]);
+                }
+            }
+            return shardData;
+        }
+
+        private void fillShardIdKeys(Set<ShardId> shardIds) {
+            int shardIdIndex = 0;
+            for (ShardId shardId : shardIds) {
+                this.shardIdToArray.putIfAbsent(shardId, shardIdIndex++);
+            }
+        }
+
+        /**
+         * A node entry, holding the state of the fetched data for a specific shard
+         * for a giving node.
+         */
+        static class NodeEntry<V> extends BaseNodeEntry {
+            private final V[] shardData;
+            private final boolean[] emptyShardResponse; // we can not rely on null entries of the shardData array,
+            // those null entries means that we need to ignore those entries. Empty responses on the other hand are
+            // actually needed in allocation/explain API response. So instead of storing full empty response object
+            // in cache, it's better to just store a boolean and create that object on the fly just before
+            // decision-making.
+            private final Predicate<V> emptyShardResponsePredicate;
+
+            NodeEntry(String nodeId, Class<V> clazz, int batchSize, Predicate<V> emptyShardResponsePredicate) {
+                super(nodeId);
+                this.shardData = (V[]) Array.newInstance(clazz, batchSize);
+                this.emptyShardResponse = new boolean[batchSize];
+                this.emptyShardResponsePredicate = emptyShardResponsePredicate;
+            }
+
+            void doneFetching(Map<ShardId, V> shardDataFromNode, Map<ShardId, Integer> shardIdKey) {
+                fillShardData(shardDataFromNode, shardIdKey);
+                super.doneFetching();
+            }
+
+            void clearShard(Integer shardIdIndex) {
+                this.shardData[shardIdIndex] = null;
+                emptyShardResponse[shardIdIndex] = false;
+            }
+
+            V[] getData() {
+                return this.shardData;
+            }
+
+            boolean[] getEmptyShardResponse() {
+                return emptyShardResponse;
+            }
+
+            private void fillShardData(Map<ShardId, V> shardDataFromNode, Map<ShardId, Integer> shardIdKey) {
+                for (Map.Entry<ShardId, V> shardData : shardDataFromNode.entrySet()) {
+                    if (shardData.getValue() != null) {
+                        ShardId shardId = shardData.getKey();
+                        if (emptyShardResponsePredicate.test(shardData.getValue())) {
+                            this.emptyShardResponse[shardIdKey.get(shardId)] = true;
+                            this.shardData[shardIdKey.get(shardId)] = null;
+                        } else {
+                            this.shardData[shardIdKey.get(shardId)] = shardData.getValue();
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
diff --git a/server/src/main/java/org/opensearch/gateway/AsyncShardFetch.java b/server/src/main/java/org/opensearch/gateway/AsyncShardFetch.java
index 3d129d4794a10..b664dd573ce67 100644
--- a/server/src/main/java/org/opensearch/gateway/AsyncShardFetch.java
+++ b/server/src/main/java/org/opensearch/gateway/AsyncShardFetch.java
@@ -82,10 +82,10 @@ public interface Lister<NodesResponse extends BaseNodesResponse<NodeResponse>, N
     protected final String type;
     protected final Map<ShardId, ShardAttributes> shardAttributesMap;
     private final Lister<BaseNodesResponse<T>, T> action;
-    private final AsyncShardFetchCache<T> cache;
+    protected final AsyncShardFetchCache<T> cache;
     private final AtomicLong round = new AtomicLong();
     private boolean closed;
-    private final String reroutingKey;
+    final String reroutingKey;
     private final Map<ShardId, Set<String>> shardToIgnoreNodes = new HashMap<>();
 
     @SuppressWarnings("unchecked")
@@ -99,7 +99,7 @@ protected AsyncShardFetch(
         this.logger = logger;
         this.type = type;
         shardAttributesMap = new HashMap<>();
-        shardAttributesMap.put(shardId, new ShardAttributes(shardId, customDataPath));
+        shardAttributesMap.put(shardId, new ShardAttributes(customDataPath));
         this.action = (Lister<BaseNodesResponse<T>, T>) action;
         this.reroutingKey = "ShardId=[" + shardId.toString() + "]";
         cache = new ShardCache<>(logger, reroutingKey, type);
@@ -120,14 +120,15 @@ protected AsyncShardFetch(
         String type,
         Map<ShardId, ShardAttributes> shardAttributesMap,
         Lister<? extends BaseNodesResponse<T>, T> action,
-        String batchId
+        String batchId,
+        AsyncShardFetchCache<T> cache
     ) {
         this.logger = logger;
         this.type = type;
         this.shardAttributesMap = shardAttributesMap;
         this.action = (Lister<BaseNodesResponse<T>, T>) action;
         this.reroutingKey = "BatchID=[" + batchId + "]";
-        cache = new ShardCache<>(logger, reroutingKey, type);
+        this.cache = cache;
     }
 
     @Override
diff --git a/server/src/main/java/org/opensearch/gateway/AsyncShardFetchCache.java b/server/src/main/java/org/opensearch/gateway/AsyncShardFetchCache.java
index 3140ceef4f3ee..2a4e6181467b0 100644
--- a/server/src/main/java/org/opensearch/gateway/AsyncShardFetchCache.java
+++ b/server/src/main/java/org/opensearch/gateway/AsyncShardFetchCache.java
@@ -48,6 +48,7 @@
  * @opensearch.internal
  */
 public abstract class AsyncShardFetchCache<K extends BaseNodeResponse> {
+
     private final Logger logger;
     private final String type;
 
diff --git a/server/src/main/java/org/opensearch/gateway/BaseGatewayShardAllocator.java b/server/src/main/java/org/opensearch/gateway/BaseGatewayShardAllocator.java
index e0831293fc7e1..eed5de65258fc 100644
--- a/server/src/main/java/org/opensearch/gateway/BaseGatewayShardAllocator.java
+++ b/server/src/main/java/org/opensearch/gateway/BaseGatewayShardAllocator.java
@@ -135,6 +135,8 @@ private void executeDecision(
         }
     }
 
+    public void allocateUnassignedBatch(String batchId, RoutingAllocation allocation) {}
+
     protected long getExpectedShardSize(ShardRouting shardRouting, RoutingAllocation allocation) {
         if (shardRouting.primary()) {
             if (shardRouting.recoverySource().getType() == RecoverySource.Type.SNAPSHOT) {
diff --git a/server/src/main/java/org/opensearch/gateway/PrimaryShardBatchAllocator.java b/server/src/main/java/org/opensearch/gateway/PrimaryShardBatchAllocator.java
index 8d222903b6f29..1979f33484d49 100644
--- a/server/src/main/java/org/opensearch/gateway/PrimaryShardBatchAllocator.java
+++ b/server/src/main/java/org/opensearch/gateway/PrimaryShardBatchAllocator.java
@@ -15,6 +15,7 @@
 import org.opensearch.cluster.routing.allocation.AllocateUnassignedDecision;
 import org.opensearch.cluster.routing.allocation.RoutingAllocation;
 import org.opensearch.gateway.AsyncShardFetch.FetchResult;
+import org.opensearch.gateway.TransportNodesGatewayStartedShardHelper.GatewayStartedShard;
 import org.opensearch.gateway.TransportNodesGatewayStartedShardHelper.NodeGatewayStartedShard;
 import org.opensearch.gateway.TransportNodesListGatewayStartedShardsBatch.NodeGatewayStartedShardsBatch;
 
@@ -132,9 +133,7 @@ private static List<NodeGatewayStartedShard> adaptToNodeShardStates(
 
         // build data for a shard from all the nodes
         nodeResponses.forEach((node, nodeGatewayStartedShardsBatch) -> {
-            TransportNodesGatewayStartedShardHelper.GatewayStartedShard shardData = nodeGatewayStartedShardsBatch
-                .getNodeGatewayStartedShardsBatch()
-                .get(unassignedShard.shardId());
+            GatewayStartedShard shardData = nodeGatewayStartedShardsBatch.getNodeGatewayStartedShardsBatch().get(unassignedShard.shardId());
             nodeShardStates.add(
                 new NodeGatewayStartedShard(
                     shardData.allocationId(),
diff --git a/server/src/main/java/org/opensearch/gateway/ReplicaShardBatchAllocator.java b/server/src/main/java/org/opensearch/gateway/ReplicaShardBatchAllocator.java
index 3459f1591b633..be7867b7823f6 100644
--- a/server/src/main/java/org/opensearch/gateway/ReplicaShardBatchAllocator.java
+++ b/server/src/main/java/org/opensearch/gateway/ReplicaShardBatchAllocator.java
@@ -56,7 +56,7 @@ public void processExistingRecoveries(RoutingAllocation allocation, List<List<Sh
                 if (shard != null && !shard.primary()) {
                     // need to iterate over all the nodes to find matching shard
                     if (shouldSkipFetchForRecovery(shard)) {
-                        ineligibleShards.add(shard);
+                        // shard should just be skipped for fetchData, no need to remove from batch
                         continue;
                     }
                     eligibleShards.add(shard);
diff --git a/server/src/main/java/org/opensearch/gateway/ShardBatchResponseFactory.java b/server/src/main/java/org/opensearch/gateway/ShardBatchResponseFactory.java
new file mode 100644
index 0000000000000..4b85ef995f1e1
--- /dev/null
+++ b/server/src/main/java/org/opensearch/gateway/ShardBatchResponseFactory.java
@@ -0,0 +1,51 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.gateway;
+
+import org.opensearch.action.support.nodes.BaseNodeResponse;
+import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.core.index.shard.ShardId;
+import org.opensearch.gateway.TransportNodesGatewayStartedShardHelper.GatewayStartedShard;
+import org.opensearch.gateway.TransportNodesListGatewayStartedShardsBatch.NodeGatewayStartedShardsBatch;
+import org.opensearch.indices.store.TransportNodesListShardStoreMetadataBatch.NodeStoreFilesMetadata;
+import org.opensearch.indices.store.TransportNodesListShardStoreMetadataBatch.NodeStoreFilesMetadataBatch;
+
+import java.util.Map;
+
+/**
+ * A factory class to create new responses of batch transport actions like
+ * {@link TransportNodesListGatewayStartedShardsBatch} or {@link org.opensearch.indices.store.TransportNodesListShardStoreMetadataBatch}
+ *
+ * @param <T> Node level response returned by batch transport actions.
+ * @param <V> Shard level metadata returned by batch transport actions.
+ */
+public class ShardBatchResponseFactory<T extends BaseNodeResponse, V> {
+    private final boolean primary;
+
+    public ShardBatchResponseFactory(boolean primary) {
+        this.primary = primary;
+    }
+
+    public T getNewResponse(DiscoveryNode node, Map<ShardId, V> shardData) {
+        if (primary) {
+            return (T) new NodeGatewayStartedShardsBatch(node, (Map<ShardId, GatewayStartedShard>) shardData);
+        } else {
+            return (T) new NodeStoreFilesMetadataBatch(node, (Map<ShardId, NodeStoreFilesMetadata>) shardData);
+        }
+    }
+
+    public Map<ShardId, V> getShardBatchData(T response) {
+        if (primary) {
+            return (Map<ShardId, V>) ((NodeGatewayStartedShardsBatch) response).getNodeGatewayStartedShardsBatch();
+        } else {
+            return (Map<ShardId, V>) ((NodeStoreFilesMetadataBatch) response).getNodeStoreFilesMetadataBatch();
+        }
+    }
+
+}
diff --git a/server/src/main/java/org/opensearch/gateway/ShardsBatchGatewayAllocator.java b/server/src/main/java/org/opensearch/gateway/ShardsBatchGatewayAllocator.java
new file mode 100644
index 0000000000000..d2de78ffac965
--- /dev/null
+++ b/server/src/main/java/org/opensearch/gateway/ShardsBatchGatewayAllocator.java
@@ -0,0 +1,721 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.gateway;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.opensearch.action.support.nodes.BaseNodeResponse;
+import org.opensearch.action.support.nodes.BaseNodesResponse;
+import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.cluster.node.DiscoveryNodes;
+import org.opensearch.cluster.routing.RerouteService;
+import org.opensearch.cluster.routing.RoutingNodes;
+import org.opensearch.cluster.routing.ShardRouting;
+import org.opensearch.cluster.routing.allocation.AllocateUnassignedDecision;
+import org.opensearch.cluster.routing.allocation.ExistingShardsAllocator;
+import org.opensearch.cluster.routing.allocation.FailedShard;
+import org.opensearch.cluster.routing.allocation.RoutingAllocation;
+import org.opensearch.common.Priority;
+import org.opensearch.common.UUIDs;
+import org.opensearch.common.inject.Inject;
+import org.opensearch.common.lease.Releasables;
+import org.opensearch.common.settings.Setting;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.concurrent.ConcurrentCollections;
+import org.opensearch.common.util.set.Sets;
+import org.opensearch.core.action.ActionListener;
+import org.opensearch.core.index.shard.ShardId;
+import org.opensearch.gateway.TransportNodesGatewayStartedShardHelper.GatewayStartedShard;
+import org.opensearch.index.store.Store;
+import org.opensearch.indices.store.ShardAttributes;
+import org.opensearch.indices.store.TransportNodesListShardStoreMetadataBatch;
+import org.opensearch.indices.store.TransportNodesListShardStoreMetadataBatch.NodeStoreFilesMetadata;
+import org.opensearch.indices.store.TransportNodesListShardStoreMetadataHelper;
+import org.opensearch.indices.store.TransportNodesListShardStoreMetadataHelper.StoreFilesMetadata;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.Spliterators;
+import java.util.concurrent.ConcurrentMap;
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import java.util.stream.StreamSupport;
+
+/**
+ * Allocator for the gateway to assign batch of shards.
+ *
+ * @opensearch.internal
+ */
+public class ShardsBatchGatewayAllocator implements ExistingShardsAllocator {
+
+    public static final String ALLOCATOR_NAME = "shards_batch_gateway_allocator";
+    private static final Logger logger = LogManager.getLogger(ShardsBatchGatewayAllocator.class);
+    private final long maxBatchSize;
+    private static final short DEFAULT_SHARD_BATCH_SIZE = 2000;
+
+    /**
+     * Number of shards we send in one batch to data nodes for fetching metadata
+     */
+    public static final Setting<Long> GATEWAY_ALLOCATOR_BATCH_SIZE = Setting.longSetting(
+        "cluster.allocator.gateway.batch_size",
+        DEFAULT_SHARD_BATCH_SIZE,
+        1,
+        10000,
+        Setting.Property.NodeScope
+    );
+
+    private final RerouteService rerouteService;
+    private final PrimaryShardBatchAllocator primaryShardBatchAllocator;
+    private final ReplicaShardBatchAllocator replicaShardBatchAllocator;
+    private Set<String> lastSeenEphemeralIds = Collections.emptySet();
+
+    // visible for testing
+    protected final ConcurrentMap<String, ShardsBatch> batchIdToStartedShardBatch = ConcurrentCollections.newConcurrentMap();
+
+    // visible for testing
+    protected final ConcurrentMap<String, ShardsBatch> batchIdToStoreShardBatch = ConcurrentCollections.newConcurrentMap();
+    private final TransportNodesListGatewayStartedShardsBatch batchStartedAction;
+    private final TransportNodesListShardStoreMetadataBatch batchStoreAction;
+
+    @Inject
+    public ShardsBatchGatewayAllocator(
+        RerouteService rerouteService,
+        TransportNodesListGatewayStartedShardsBatch batchStartedAction,
+        TransportNodesListShardStoreMetadataBatch batchStoreAction,
+        Settings settings
+    ) {
+        this.rerouteService = rerouteService;
+        this.primaryShardBatchAllocator = new InternalPrimaryBatchShardAllocator();
+        this.replicaShardBatchAllocator = new InternalReplicaBatchShardAllocator();
+        this.batchStartedAction = batchStartedAction;
+        this.batchStoreAction = batchStoreAction;
+        this.maxBatchSize = GATEWAY_ALLOCATOR_BATCH_SIZE.get(settings);
+    }
+
+    @Override
+    public void cleanCaches() {
+        Stream.of(batchIdToStartedShardBatch, batchIdToStoreShardBatch).forEach(b -> {
+            Releasables.close(b.values().stream().map(shardsBatch -> shardsBatch.asyncBatch).collect(Collectors.toList()));
+            b.clear();
+        });
+    }
+
+    // for tests
+    protected ShardsBatchGatewayAllocator() {
+        this.rerouteService = null;
+        this.batchStartedAction = null;
+        this.primaryShardBatchAllocator = null;
+        this.batchStoreAction = null;
+        this.replicaShardBatchAllocator = null;
+        this.maxBatchSize = DEFAULT_SHARD_BATCH_SIZE;
+    }
+
+    // for tests
+
+    @Override
+    public int getNumberOfInFlightFetches() {
+        int count = 0;
+        for (ShardsBatch batch : batchIdToStartedShardBatch.values()) {
+            count += (batch.getNumberOfInFlightFetches() * batch.getBatchedShards().size());
+        }
+        for (ShardsBatch batch : batchIdToStoreShardBatch.values()) {
+            count += (batch.getNumberOfInFlightFetches() * batch.getBatchedShards().size());
+        }
+
+        return count;
+    }
+
+    @Override
+    public void applyStartedShards(final List<ShardRouting> startedShards, final RoutingAllocation allocation) {
+        for (ShardRouting startedShard : startedShards) {
+            safelyRemoveShardFromBothBatch(startedShard);
+        }
+    }
+
+    @Override
+    public void applyFailedShards(final List<FailedShard> failedShards, final RoutingAllocation allocation) {
+        for (FailedShard failedShard : failedShards) {
+            safelyRemoveShardFromBothBatch(failedShard.getRoutingEntry());
+        }
+    }
+
+    @Override
+    public void beforeAllocation(final RoutingAllocation allocation) {
+        assert primaryShardBatchAllocator != null;
+        assert replicaShardBatchAllocator != null;
+        ensureAsyncFetchStorePrimaryRecency(allocation);
+    }
+
+    @Override
+    public void afterPrimariesBeforeReplicas(RoutingAllocation allocation) {
+        assert replicaShardBatchAllocator != null;
+        List<List<ShardRouting>> storedShardBatches = batchIdToStoreShardBatch.values()
+            .stream()
+            .map(ShardsBatch::getBatchedShardRoutings)
+            .collect(Collectors.toList());
+        if (allocation.routingNodes().hasInactiveShards()) {
+            // cancel existing recoveries if we have a better match
+            replicaShardBatchAllocator.processExistingRecoveries(allocation, storedShardBatches);
+        }
+    }
+
+    @Override
+    public void allocateUnassigned(
+        ShardRouting shardRouting,
+        RoutingAllocation allocation,
+        UnassignedAllocationHandler unassignedAllocationHandler
+    ) {
+        throw new UnsupportedOperationException("ShardsBatchGatewayAllocator does not support allocating unassigned shards");
+    }
+
+    @Override
+    public void allocateAllUnassignedShards(final RoutingAllocation allocation, boolean primary) {
+
+        assert primaryShardBatchAllocator != null;
+        assert replicaShardBatchAllocator != null;
+        innerAllocateUnassignedBatch(allocation, primaryShardBatchAllocator, replicaShardBatchAllocator, primary);
+    }
+
+    protected void innerAllocateUnassignedBatch(
+        RoutingAllocation allocation,
+        PrimaryShardBatchAllocator primaryBatchShardAllocator,
+        ReplicaShardBatchAllocator replicaBatchShardAllocator,
+        boolean primary
+    ) {
+        // create batches for unassigned shards
+        Set<String> batchesToAssign = createAndUpdateBatches(allocation, primary);
+        if (batchesToAssign.isEmpty()) {
+            return;
+        }
+        if (primary) {
+            batchIdToStartedShardBatch.values()
+                .stream()
+                .filter(batch -> batchesToAssign.contains(batch.batchId))
+                .forEach(
+                    shardsBatch -> primaryBatchShardAllocator.allocateUnassignedBatch(shardsBatch.getBatchedShardRoutings(), allocation)
+                );
+        } else {
+            batchIdToStoreShardBatch.values()
+                .stream()
+                .filter(batch -> batchesToAssign.contains(batch.batchId))
+                .forEach(batch -> replicaBatchShardAllocator.allocateUnassignedBatch(batch.getBatchedShardRoutings(), allocation));
+        }
+    }
+
+    // visible for testing
+    protected Set<String> createAndUpdateBatches(RoutingAllocation allocation, boolean primary) {
+        Set<String> batchesToBeAssigned = new HashSet<>();
+        RoutingNodes.UnassignedShards unassigned = allocation.routingNodes().unassigned();
+        ConcurrentMap<String, ShardsBatch> currentBatches = primary ? batchIdToStartedShardBatch : batchIdToStoreShardBatch;
+        // get all batched shards
+        Map<ShardId, String> currentBatchedShards = new HashMap<>();
+        for (Map.Entry<String, ShardsBatch> batchEntry : currentBatches.entrySet()) {
+            batchEntry.getValue().getBatchedShards().forEach(shardId -> currentBatchedShards.put(shardId, batchEntry.getKey()));
+        }
+
+        Set<ShardRouting> newShardsToBatch = Sets.newHashSet();
+        Set<ShardId> batchedShardsToAssign = Sets.newHashSet();
+        // add all unassigned shards to the batch if they are not already in a batch
+        unassigned.forEach(shardRouting -> {
+            if ((currentBatchedShards.containsKey(shardRouting.shardId()) == false) && (shardRouting.primary() == primary)) {
+                assert shardRouting.unassigned();
+                newShardsToBatch.add(shardRouting);
+            }
+            // if shard is already batched update to latest shardRouting information in the batches
+            // Replica shard assignment can be cancelled if we get a better match. These ShardRouting objects also
+            // store other information like relocating node, targetRelocatingShard etc. And it can be updated after
+            // batches are created. If we don't update the ShardRouting object, stale data would be passed from the
+            // batch. This stale data can end up creating a same decision which has already been taken, and we'll see
+            // failure in executeDecision of BaseGatewayShardAllocator. Previous non-batch mode flow also used to
+            // pass ShardRouting object directly from unassignedIterator, so we're following the same behaviour.
+            else if (shardRouting.primary() == primary) {
+                String batchId = currentBatchedShards.get(shardRouting.shardId());
+                batchesToBeAssigned.add(batchId);
+                currentBatches.get(batchId).batchInfo.get(shardRouting.shardId()).setShardRouting(shardRouting);
+                batchedShardsToAssign.add(shardRouting.shardId());
+            }
+        });
+
+        allocation.routingNodes().forEach(routingNode -> routingNode.getInitializingShards().forEach(shardRouting -> {
+            if (currentBatchedShards.containsKey(shardRouting.shardId()) && shardRouting.primary() == primary) {
+                batchedShardsToAssign.add(shardRouting.shardId());
+                // Set updated shard routing in batch if it already exists
+                String batchId = currentBatchedShards.get(shardRouting.shardId());
+                currentBatches.get(batchId).batchInfo.get(shardRouting.shardId()).setShardRouting(shardRouting);
+            }
+        }));
+
+        refreshShardBatches(currentBatches, batchedShardsToAssign, primary);
+
+        Iterator<ShardRouting> iterator = newShardsToBatch.iterator();
+        assert maxBatchSize > 0 : "Shards batch size must be greater than 0";
+
+        long batchSize = maxBatchSize;
+        Map<ShardId, ShardEntry> perBatchShards = new HashMap<>();
+        while (iterator.hasNext()) {
+            ShardRouting currentShard = iterator.next();
+            ShardEntry shardEntry = new ShardEntry(
+                new ShardAttributes(
+                    IndexMetadata.INDEX_DATA_PATH_SETTING.get(allocation.metadata().index(currentShard.index()).getSettings())
+                ),
+                currentShard
+            );
+            perBatchShards.put(currentShard.shardId(), shardEntry);
+            batchSize--;
+            iterator.remove();
+            // add to batch if batch size full or last shard in unassigned list
+            if (batchSize == 0 || iterator.hasNext() == false) {
+                String batchUUId = UUIDs.base64UUID();
+                ShardsBatch shardsBatch = new ShardsBatch(batchUUId, perBatchShards, primary);
+                // add the batch to list of current batches
+                addBatch(shardsBatch, primary);
+                batchesToBeAssigned.add(batchUUId);
+                perBatchShards.clear();
+                batchSize = maxBatchSize;
+            }
+        }
+        return batchesToBeAssigned;
+    }
+
+    private void refreshShardBatches(
+        ConcurrentMap<String, ShardsBatch> currentBatches,
+        Set<ShardId> batchedShardsToAssign,
+        boolean primary
+    ) {
+        // cleanup shard from batches if they are not present in unassigned list from allocation object. This is
+        // needed as AllocationService.reroute can also be called directly by API flows for example DeleteIndices.
+        // So, as part of calling reroute, those shards will be removed from allocation object. It'll handle the
+        // scenarios where shards can be removed from unassigned list without "start" or "failed" event.
+        for (Map.Entry<String, ShardsBatch> batchEntry : currentBatches.entrySet()) {
+            Iterator<ShardId> shardIdIterator = batchEntry.getValue().getBatchedShards().iterator();
+            while (shardIdIterator.hasNext()) {
+                ShardId shardId = shardIdIterator.next();
+                if (batchedShardsToAssign.contains(shardId) == false) {
+                    shardIdIterator.remove();
+                    batchEntry.getValue().clearShardFromCache(shardId);
+                }
+            }
+            ConcurrentMap<String, ShardsBatch> batches = primary ? batchIdToStartedShardBatch : batchIdToStoreShardBatch;
+            deleteBatchIfEmpty(batches, batchEntry.getValue().getBatchId());
+        }
+    }
+
+    private void addBatch(ShardsBatch shardsBatch, boolean primary) {
+        ConcurrentMap<String, ShardsBatch> batches = primary ? batchIdToStartedShardBatch : batchIdToStoreShardBatch;
+        if (batches.containsKey(shardsBatch.getBatchId())) {
+            throw new IllegalStateException("Batch already exists. BatchId = " + shardsBatch.getBatchId());
+        }
+        batches.put(shardsBatch.getBatchId(), shardsBatch);
+    }
+
+    /**
+     * Safely remove a shard from the appropriate batch depending on if it is primary or replica
+     * If the shard is not in a batch, this is a no-op.
+     * Cleans the batch if it is empty after removing the shard.
+     * This method should be called when removing the shard from the batch instead {@link ShardsBatch#removeFromBatch(ShardRouting)}
+     * so that we can clean up the batch if it is empty and release the fetching resources
+     *
+     * @param shardRouting shard to be removed
+     * @param primary from which batch shard needs to be removed
+     */
+    protected void safelyRemoveShardFromBatch(ShardRouting shardRouting, boolean primary) {
+        String batchId = primary ? getBatchId(shardRouting, true) : getBatchId(shardRouting, false);
+        if (batchId == null) {
+            logger.debug("Shard[{}] is not batched", shardRouting);
+            return;
+        }
+        ConcurrentMap<String, ShardsBatch> batches = primary ? batchIdToStartedShardBatch : batchIdToStoreShardBatch;
+        ShardsBatch batch = batches.get(batchId);
+        batch.removeFromBatch(shardRouting);
+        deleteBatchIfEmpty(batches, batchId);
+    }
+
+    /**
+     * Safely remove shard from both the batches irrespective of its primary or replica,
+     * For the corresponding shardId. The method intends to clean up the batch if it is empty
+     * after removing the shard
+     * @param shardRouting shard to remove
+     */
+    protected void safelyRemoveShardFromBothBatch(ShardRouting shardRouting) {
+        safelyRemoveShardFromBatch(shardRouting, true);
+        safelyRemoveShardFromBatch(shardRouting, false);
+    }
+
+    private void deleteBatchIfEmpty(ConcurrentMap<String, ShardsBatch> batches, String batchId) {
+        if (batches.containsKey(batchId)) {
+            ShardsBatch batch = batches.get(batchId);
+            if (batch.getBatchedShards().isEmpty()) {
+                Releasables.close(batch.getAsyncFetcher());
+                batches.remove(batchId);
+            }
+        }
+    }
+
+    protected String getBatchId(ShardRouting shardRouting, boolean primary) {
+        ConcurrentMap<String, ShardsBatch> batches = primary ? batchIdToStartedShardBatch : batchIdToStoreShardBatch;
+
+        return batches.entrySet()
+            .stream()
+            .filter(entry -> entry.getValue().getBatchedShards().contains(shardRouting.shardId()))
+            .findFirst()
+            .map(Map.Entry::getKey)
+            .orElse(null);
+    }
+
+    @Override
+    public AllocateUnassignedDecision explainUnassignedShardAllocation(ShardRouting unassignedShard, RoutingAllocation routingAllocation) {
+        assert unassignedShard.unassigned();
+        assert routingAllocation.debugDecision();
+        if (getBatchId(unassignedShard, unassignedShard.primary()) == null) {
+            createAndUpdateBatches(routingAllocation, unassignedShard.primary());
+        }
+        assert getBatchId(unassignedShard, unassignedShard.primary()) != null;
+        if (unassignedShard.primary()) {
+            assert primaryShardBatchAllocator != null;
+            return primaryShardBatchAllocator.makeAllocationDecision(unassignedShard, routingAllocation, logger);
+        } else {
+            assert replicaShardBatchAllocator != null;
+            return replicaShardBatchAllocator.makeAllocationDecision(unassignedShard, routingAllocation, logger);
+        }
+    }
+
+    /**
+     * Clear the fetched data for the primary to ensure we do not cancel recoveries based on excessively stale data.
+     */
+    private void ensureAsyncFetchStorePrimaryRecency(RoutingAllocation allocation) {
+        DiscoveryNodes nodes = allocation.nodes();
+        if (hasNewNodes(nodes)) {
+            final Set<String> newEphemeralIds = StreamSupport.stream(Spliterators.spliterator(nodes.getDataNodes().entrySet(), 0), false)
+                .map(node -> node.getValue().getEphemeralId())
+                .collect(Collectors.toSet());
+            // Invalidate the cache if a data node has been added to the cluster. This ensures that we do not cancel a recovery if a node
+            // drops out, we fetch the shard data, then some indexing happens and then the node rejoins the cluster again. There are other
+            // ways we could decide to cancel a recovery based on stale data (e.g. changing allocation filters or a primary failure) but
+            // making the wrong decision here is not catastrophic so we only need to cover the common case.
+
+            logger.trace(
+                () -> new ParameterizedMessage(
+                    "new nodes {} found, clearing primary async-fetch-store cache",
+                    Sets.difference(newEphemeralIds, lastSeenEphemeralIds)
+                )
+            );
+            batchIdToStoreShardBatch.values().forEach(batch -> clearCacheForBatchPrimary(batch, allocation));
+
+            // recalc to also (lazily) clear out old nodes.
+            this.lastSeenEphemeralIds = newEphemeralIds;
+        }
+    }
+
+    private static void clearCacheForBatchPrimary(ShardsBatch batch, RoutingAllocation allocation) {
+        // We need to clear the cache for the primary shard to ensure we do not cancel recoveries based on excessively
+        // stale data. We do this by clearing the cache of nodes for all the active primaries of replicas in the current batch.
+        // Although this flow can be optimized by only clearing the cache for the primary shard but currently
+        // when we want to fetch data we do for complete node, for doing this a new fetch flow will also handle just
+        // fetching the data for a single shard on the node and fill that up in our cache
+        // Opened issue #13352 - to track the improvement
+        List<ShardRouting> primaries = batch.getBatchedShards()
+            .stream()
+            .map(allocation.routingNodes()::activePrimary)
+            .filter(Objects::nonNull)
+            .collect(Collectors.toList());
+        AsyncShardBatchFetch<? extends BaseNodeResponse, ?> fetch = batch.getAsyncFetcher();
+        primaries.forEach(shardRouting -> fetch.clearCacheForNode(shardRouting.currentNodeId()));
+    }
+
+    private boolean hasNewNodes(DiscoveryNodes nodes) {
+        for (final DiscoveryNode node : nodes.getDataNodes().values()) {
+            if (lastSeenEphemeralIds.contains(node.getEphemeralId()) == false) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    class InternalBatchAsyncFetch<T extends BaseNodeResponse, V> extends AsyncShardBatchFetch<T, V> {
+        InternalBatchAsyncFetch(
+            Logger logger,
+            String type,
+            Map<ShardId, ShardAttributes> map,
+            AsyncShardFetch.Lister<? extends BaseNodesResponse<T>, T> action,
+            String batchUUId,
+            Class<V> clazz,
+            V emptyShardResponse,
+            Predicate<V> emptyShardResponsePredicate,
+            ShardBatchResponseFactory<T, V> responseFactory
+        ) {
+            super(logger, type, map, action, batchUUId, clazz, emptyShardResponse, emptyShardResponsePredicate, responseFactory);
+        }
+
+        @Override
+        protected void reroute(String reroutingKey, String reason) {
+            logger.trace("{} scheduling reroute for {}", reroutingKey, reason);
+            assert rerouteService != null;
+            rerouteService.reroute(
+                "async_shard_batch_fetch",
+                Priority.HIGH,
+                ActionListener.wrap(
+                    r -> logger.trace("{} scheduled reroute completed for {}", reroutingKey, reason),
+                    e -> logger.debug(new ParameterizedMessage("{} scheduled reroute failed for {}", reroutingKey, reason), e)
+                )
+            );
+        }
+    }
+
+    class InternalPrimaryBatchShardAllocator extends PrimaryShardBatchAllocator {
+
+        @Override
+        @SuppressWarnings("unchecked")
+        protected AsyncShardFetch.FetchResult<TransportNodesListGatewayStartedShardsBatch.NodeGatewayStartedShardsBatch> fetchData(
+            List<ShardRouting> eligibleShards,
+            List<ShardRouting> inEligibleShards,
+            RoutingAllocation allocation
+        ) {
+            return (AsyncShardFetch.FetchResult<
+                TransportNodesListGatewayStartedShardsBatch.NodeGatewayStartedShardsBatch>) fetchDataAndCleanIneligibleShards(
+                    eligibleShards,
+                    inEligibleShards,
+                    allocation
+                );
+        }
+
+    }
+
+    class InternalReplicaBatchShardAllocator extends ReplicaShardBatchAllocator {
+        @Override
+        @SuppressWarnings("unchecked")
+        protected AsyncShardFetch.FetchResult<TransportNodesListShardStoreMetadataBatch.NodeStoreFilesMetadataBatch> fetchData(
+            List<ShardRouting> eligibleShards,
+            List<ShardRouting> inEligibleShards,
+            RoutingAllocation allocation
+        ) {
+            return (AsyncShardFetch.FetchResult<
+                TransportNodesListShardStoreMetadataBatch.NodeStoreFilesMetadataBatch>) fetchDataAndCleanIneligibleShards(
+                    eligibleShards,
+                    inEligibleShards,
+                    allocation
+                );
+        }
+
+        @Override
+        protected boolean hasInitiatedFetching(ShardRouting shard) {
+            String batchId = getBatchId(shard, shard.primary());
+            return batchId != null;
+        }
+    }
+
+    AsyncShardFetch.FetchResult<? extends BaseNodeResponse> fetchDataAndCleanIneligibleShards(
+        List<ShardRouting> eligibleShards,
+        List<ShardRouting> inEligibleShards,
+        RoutingAllocation allocation
+    ) {
+        // get batch id for anyone given shard. We are assuming all shards will have same batchId
+        ShardRouting shardRouting = eligibleShards.iterator().hasNext() ? eligibleShards.iterator().next() : null;
+        shardRouting = shardRouting == null && inEligibleShards.iterator().hasNext() ? inEligibleShards.iterator().next() : shardRouting;
+        if (shardRouting == null) {
+            return new AsyncShardFetch.FetchResult<>(null, Collections.emptyMap());
+        }
+        String batchId = getBatchId(shardRouting, shardRouting.primary());
+        if (batchId == null) {
+            logger.debug("Shard {} has no batch id", shardRouting);
+            throw new IllegalStateException("Shard " + shardRouting + " has no batch id. Shard should batched before fetching");
+        }
+        ConcurrentMap<String, ShardsBatch> batches = shardRouting.primary() ? batchIdToStartedShardBatch : batchIdToStoreShardBatch;
+        if (batches.containsKey(batchId) == false) {
+            logger.debug("Batch {} has no shards batch", batchId);
+            throw new IllegalStateException("Batch " + batchId + " has no shards batch");
+        }
+
+        ShardsBatch shardsBatch = batches.get(batchId);
+        // remove in eligible shards which allocator is not responsible for
+        inEligibleShards.forEach(sr -> safelyRemoveShardFromBatch(sr, sr.primary()));
+
+        if (shardsBatch.getBatchedShards().isEmpty() && eligibleShards.isEmpty()) {
+            logger.debug("Batch {} is empty", batchId);
+            return new AsyncShardFetch.FetchResult<>(null, Collections.emptyMap());
+        }
+        Map<ShardId, Set<String>> shardToIgnoreNodes = new HashMap<>();
+        for (ShardId shardId : shardsBatch.asyncBatch.shardAttributesMap.keySet()) {
+            shardToIgnoreNodes.put(shardId, allocation.getIgnoreNodes(shardId));
+        }
+        AsyncShardBatchFetch<? extends BaseNodeResponse, ?> asyncFetcher = shardsBatch.getAsyncFetcher();
+        AsyncShardFetch.FetchResult<? extends BaseNodeResponse> fetchResult = asyncFetcher.fetchData(
+            allocation.nodes(),
+            shardToIgnoreNodes
+        );
+        if (fetchResult.hasData()) {
+            fetchResult.processAllocation(allocation);
+        }
+
+        return fetchResult;
+    }
+
+    /**
+     * Holds information about a batch of shards to be allocated.
+     * Async fetcher is used to fetch the data for the batch.
+     * <p>
+     * Visible for testing
+     */
+    public class ShardsBatch {
+        private final String batchId;
+        private final boolean primary;
+
+        private final InternalBatchAsyncFetch<? extends BaseNodeResponse, ?> asyncBatch;
+
+        private final Map<ShardId, ShardEntry> batchInfo;
+
+        public ShardsBatch(String batchId, Map<ShardId, ShardEntry> shardsWithInfo, boolean primary) {
+            this.batchId = batchId;
+            this.batchInfo = new HashMap<>(shardsWithInfo);
+            // create a ShardId -> customDataPath map for async fetch
+            Map<ShardId, ShardAttributes> shardIdsMap = batchInfo.entrySet()
+                .stream()
+                .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().getShardAttributes()));
+            this.primary = primary;
+            if (this.primary) {
+                asyncBatch = new InternalBatchAsyncFetch<>(
+                    logger,
+                    "batch_shards_started",
+                    shardIdsMap,
+                    batchStartedAction,
+                    batchId,
+                    GatewayStartedShard.class,
+                    new GatewayStartedShard(null, false, null, null),
+                    GatewayStartedShard::isEmpty,
+                    new ShardBatchResponseFactory<>(true)
+                );
+            } else {
+                asyncBatch = new InternalBatchAsyncFetch<>(
+                    logger,
+                    "batch_shards_store",
+                    shardIdsMap,
+                    batchStoreAction,
+                    batchId,
+                    NodeStoreFilesMetadata.class,
+                    new NodeStoreFilesMetadata(new StoreFilesMetadata(null, Store.MetadataSnapshot.EMPTY, Collections.emptyList()), null),
+                    NodeStoreFilesMetadata::isEmpty,
+                    new ShardBatchResponseFactory<>(false)
+                );
+            }
+        }
+
+        protected void removeShard(ShardId shardId) {
+            this.batchInfo.remove(shardId);
+        }
+
+        private TransportNodesListShardStoreMetadataBatch.NodeStoreFilesMetadata buildEmptyReplicaShardResponse() {
+            return new TransportNodesListShardStoreMetadataBatch.NodeStoreFilesMetadata(
+                new TransportNodesListShardStoreMetadataHelper.StoreFilesMetadata(
+                    null,
+                    Store.MetadataSnapshot.EMPTY,
+                    Collections.emptyList()
+                ),
+                null
+            );
+        }
+
+        private void removeFromBatch(ShardRouting shard) {
+            removeShard(shard.shardId());
+            clearShardFromCache(shard.shardId());
+            // assert that fetcher and shards are the same as batched shards
+            assert batchInfo.size() == asyncBatch.shardAttributesMap.size() : "Shards size is not equal to fetcher size";
+        }
+
+        private void clearShardFromCache(ShardId shardId) {
+            asyncBatch.clearShard(shardId);
+        }
+
+        public List<ShardRouting> getBatchedShardRoutings() {
+            return batchInfo.values().stream().map(ShardEntry::getShardRouting).collect(Collectors.toList());
+        }
+
+        public Set<ShardId> getBatchedShards() {
+            return batchInfo.keySet();
+        }
+
+        public String getBatchId() {
+            return batchId;
+        }
+
+        public AsyncShardBatchFetch<? extends BaseNodeResponse, ?> getAsyncFetcher() {
+            return asyncBatch;
+        }
+
+        public int getNumberOfInFlightFetches() {
+            return asyncBatch.getNumberOfInFlightFetches();
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) {
+                return true;
+            }
+            if (o instanceof ShardsBatch == false) {
+                return false;
+            }
+            ShardsBatch shardsBatch = (ShardsBatch) o;
+            return batchId.equals(shardsBatch.getBatchId()) && batchInfo.keySet().equals(shardsBatch.getBatchedShards());
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(batchId);
+        }
+
+        @Override
+        public String toString() {
+            return "batchId: " + batchId;
+        }
+
+    }
+
+    /**
+     * Holds information about a shard to be allocated in a batch.
+     */
+    static class ShardEntry {
+
+        private final ShardAttributes shardAttributes;
+
+        private ShardRouting shardRouting;
+
+        public ShardEntry(ShardAttributes shardAttributes, ShardRouting shardRouting) {
+            this.shardAttributes = shardAttributes;
+            this.shardRouting = shardRouting;
+        }
+
+        public ShardRouting getShardRouting() {
+            return shardRouting;
+        }
+
+        public ShardAttributes getShardAttributes() {
+            return shardAttributes;
+        }
+
+        public ShardEntry setShardRouting(ShardRouting shardRouting) {
+            this.shardRouting = shardRouting;
+            return this;
+        }
+    }
+
+    public int getNumberOfStartedShardBatches() {
+        return batchIdToStartedShardBatch.size();
+    }
+
+    public int getNumberOfStoreShardBatches() {
+        return batchIdToStoreShardBatch.size();
+    }
+}
diff --git a/server/src/main/java/org/opensearch/gateway/TransportNodesGatewayStartedShardHelper.java b/server/src/main/java/org/opensearch/gateway/TransportNodesGatewayStartedShardHelper.java
index 27cce76b1b694..2ddae1d8410c9 100644
--- a/server/src/main/java/org/opensearch/gateway/TransportNodesGatewayStartedShardHelper.java
+++ b/server/src/main/java/org/opensearch/gateway/TransportNodesGatewayStartedShardHelper.java
@@ -42,6 +42,8 @@
  * @opensearch.internal
  */
 public class TransportNodesGatewayStartedShardHelper {
+    public static final String INDEX_NOT_FOUND = "node doesn't have meta data for index";
+
     public static GatewayStartedShard getShardInfoOnLocalNode(
         Logger logger,
         final ShardId shardId,
@@ -72,7 +74,7 @@ public static GatewayStartedShard getShardInfoOnLocalNode(
                         customDataPath = new IndexSettings(metadata, settings).customDataPath();
                     } else {
                         logger.trace("{} node doesn't have meta data for the requests index", shardId);
-                        throw new OpenSearchException("node doesn't have meta data for index " + shardId.getIndex());
+                        throw new OpenSearchException(INDEX_NOT_FOUND + " " + shardId.getIndex());
                     }
                 }
                 // we don't have an open shard on the store, validate the files on disk are openable
@@ -230,6 +232,13 @@ public String toString() {
             buf.append("]");
             return buf.toString();
         }
+
+        public static boolean isEmpty(GatewayStartedShard gatewayStartedShard) {
+            return gatewayStartedShard.allocationId() == null
+                && gatewayStartedShard.primary() == false
+                && gatewayStartedShard.storeException() == null
+                && gatewayStartedShard.replicationCheckpoint() == null;
+        }
     }
 
     /**
diff --git a/server/src/main/java/org/opensearch/gateway/TransportNodesListGatewayStartedShardsBatch.java b/server/src/main/java/org/opensearch/gateway/TransportNodesListGatewayStartedShardsBatch.java
index dc5d85b17bc32..89362988b4d85 100644
--- a/server/src/main/java/org/opensearch/gateway/TransportNodesListGatewayStartedShardsBatch.java
+++ b/server/src/main/java/org/opensearch/gateway/TransportNodesListGatewayStartedShardsBatch.java
@@ -8,7 +8,6 @@
 
 package org.opensearch.gateway;
 
-import org.opensearch.OpenSearchException;
 import org.opensearch.action.ActionType;
 import org.opensearch.action.FailedNodeException;
 import org.opensearch.action.support.ActionFilters;
@@ -27,7 +26,6 @@
 import org.opensearch.core.index.shard.ShardId;
 import org.opensearch.core.xcontent.NamedXContentRegistry;
 import org.opensearch.env.NodeEnvironment;
-import org.opensearch.gateway.TransportNodesGatewayStartedShardHelper.GatewayStartedShard;
 import org.opensearch.indices.IndicesService;
 import org.opensearch.indices.store.ShardAttributes;
 import org.opensearch.threadpool.ThreadPool;
@@ -40,6 +38,8 @@
 import java.util.Map;
 import java.util.Objects;
 
+import static org.opensearch.gateway.TransportNodesGatewayStartedShardHelper.GatewayStartedShard;
+import static org.opensearch.gateway.TransportNodesGatewayStartedShardHelper.INDEX_NOT_FOUND;
 import static org.opensearch.gateway.TransportNodesGatewayStartedShardHelper.getShardInfoOnLocalNode;
 
 /**
@@ -136,8 +136,10 @@ protected NodesGatewayStartedShardsBatch newResponse(
     @Override
     protected NodeGatewayStartedShardsBatch nodeOperation(NodeRequest request) {
         Map<ShardId, GatewayStartedShard> shardsOnNode = new HashMap<>();
-        for (ShardAttributes shardAttr : request.shardAttributes.values()) {
-            final ShardId shardId = shardAttr.getShardId();
+        // NOTE : If we ever change this for loop to run in parallel threads, we should re-visit the exception
+        // handling in AsyncShardBatchFetch class.
+        for (Map.Entry<ShardId, ShardAttributes> shardAttr : request.shardAttributes.entrySet()) {
+            final ShardId shardId = shardAttr.getKey();
             try {
                 shardsOnNode.put(
                     shardId,
@@ -147,16 +149,19 @@ protected NodeGatewayStartedShardsBatch nodeOperation(NodeRequest request) {
                         namedXContentRegistry,
                         nodeEnv,
                         indicesService,
-                        shardAttr.getCustomDataPath(),
+                        shardAttr.getValue().getCustomDataPath(),
                         settings,
                         clusterService
                     )
                 );
             } catch (Exception e) {
-                shardsOnNode.put(
-                    shardId,
-                    new GatewayStartedShard(null, false, null, new OpenSearchException("failed to load started shards", e))
-                );
+                // should return null in case of known exceptions being returned from getShardInfoOnLocalNode method.
+                if (e instanceof IllegalStateException || e.getMessage().contains(INDEX_NOT_FOUND) || e instanceof IOException) {
+                    shardsOnNode.put(shardId, null);
+                } else {
+                    // return actual exception as it is for unknown exceptions
+                    shardsOnNode.put(shardId, new GatewayStartedShard(null, false, null, e));
+                }
             }
         }
         return new NodeGatewayStartedShardsBatch(clusterService.localNode(), shardsOnNode);
@@ -264,13 +269,26 @@ public Map<ShardId, GatewayStartedShard> getNodeGatewayStartedShardsBatch() {
 
         public NodeGatewayStartedShardsBatch(StreamInput in) throws IOException {
             super(in);
-            this.nodeGatewayStartedShardsBatch = in.readMap(ShardId::new, GatewayStartedShard::new);
+            this.nodeGatewayStartedShardsBatch = in.readMap(ShardId::new, i -> {
+                if (i.readBoolean()) {
+                    return new GatewayStartedShard(i);
+                } else {
+                    return null;
+                }
+            });
         }
 
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
-            out.writeMap(nodeGatewayStartedShardsBatch, (o, k) -> k.writeTo(o), (o, v) -> v.writeTo(o));
+            out.writeMap(nodeGatewayStartedShardsBatch, (o, k) -> k.writeTo(o), (o, v) -> {
+                if (v != null) {
+                    o.writeBoolean(true);
+                    v.writeTo(o);
+                } else {
+                    o.writeBoolean(false);
+                }
+            });
         }
 
         public NodeGatewayStartedShardsBatch(DiscoveryNode node, Map<ShardId, GatewayStartedShard> nodeGatewayStartedShardsBatch) {
diff --git a/server/src/main/java/org/opensearch/gateway/remote/IndexMetadataUploadListener.java b/server/src/main/java/org/opensearch/gateway/remote/IndexMetadataUploadListener.java
new file mode 100644
index 0000000000000..7d5fe8140e1e1
--- /dev/null
+++ b/server/src/main/java/org/opensearch/gateway/remote/IndexMetadataUploadListener.java
@@ -0,0 +1,58 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.gateway.remote;
+
+import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.core.action.ActionListener;
+import org.opensearch.threadpool.ThreadPool;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.ExecutorService;
+
+/**
+ * Hook for running code that needs to be executed before the upload of index metadata. Here we have introduced a hook
+ * for index creation (also triggerred after enabling the remote cluster statement for the first time). The listener
+ * is intended to be run in parallel and async with the index metadata upload.
+ *
+ * @opensearch.internal
+ */
+public abstract class IndexMetadataUploadListener {
+
+    private final ExecutorService executorService;
+
+    public IndexMetadataUploadListener(ThreadPool threadPool, String threadPoolName) {
+        Objects.requireNonNull(threadPool);
+        Objects.requireNonNull(threadPoolName);
+        assert ThreadPool.THREAD_POOL_TYPES.containsKey(threadPoolName) && ThreadPool.Names.SAME.equals(threadPoolName) == false;
+        this.executorService = threadPool.executor(threadPoolName);
+    }
+
+    /**
+     * Runs before the new index upload of index metadata (or first time upload). The caller is expected to trigger
+     * onSuccess or onFailure of the {@code ActionListener}.
+     *
+     * @param indexMetadataList list of index metadata of new indexes (or first time index metadata upload).
+     * @param actionListener    listener to be invoked on success or failure.
+     */
+    public final void onUpload(
+        List<IndexMetadata> indexMetadataList,
+        Map<String, IndexMetadata> prevIndexMetadataByName,
+        ActionListener<Void> actionListener
+    ) {
+        executorService.execute(() -> doOnUpload(indexMetadataList, prevIndexMetadataByName, actionListener));
+    }
+
+    protected abstract void doOnUpload(
+        List<IndexMetadata> indexMetadataList,
+        Map<String, IndexMetadata> prevIndexMetadataByName,
+        ActionListener<Void> actionListener
+    );
+}
diff --git a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
index c892b475d71da..eaf607564185c 100644
--- a/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
+++ b/server/src/main/java/org/opensearch/gateway/remote/RemoteClusterStateService.java
@@ -160,6 +160,7 @@ public class RemoteClusterStateService implements Closeable {
     private final Settings settings;
     private final LongSupplier relativeTimeNanosSupplier;
     private final ThreadPool threadpool;
+    private final List<IndexMetadataUploadListener> indexMetadataUploadListeners;
     private BlobStoreRepository blobStoreRepository;
     private BlobStoreTransferService blobStoreTransferService;
     private volatile TimeValue slowWriteLoggingThreshold;
@@ -177,6 +178,7 @@ public class RemoteClusterStateService implements Closeable {
     // ToXContent Params with gateway mode.
     // We are using gateway context mode to persist all custom metadata.
     public static final ToXContent.Params FORMAT_PARAMS;
+
     static {
         Map<String, String> params = new HashMap<>(1);
         params.put(Metadata.CONTEXT_MODE_PARAM, Metadata.CONTEXT_MODE_GATEWAY);
@@ -189,7 +191,8 @@ public RemoteClusterStateService(
         Settings settings,
         ClusterSettings clusterSettings,
         LongSupplier relativeTimeNanosSupplier,
-        ThreadPool threadPool
+        ThreadPool threadPool,
+        List<IndexMetadataUploadListener> indexMetadataUploadListeners
     ) {
         assert isRemoteStoreClusterStateEnabled(settings) : "Remote cluster state is not enabled";
         this.nodeId = nodeId;
@@ -206,6 +209,7 @@ public RemoteClusterStateService(
         clusterSettings.addSettingsUpdateConsumer(GLOBAL_METADATA_UPLOAD_TIMEOUT_SETTING, this::setGlobalMetadataUploadTimeout);
         clusterSettings.addSettingsUpdateConsumer(METADATA_MANIFEST_UPLOAD_TIMEOUT_SETTING, this::setMetadataManifestUploadTimeout);
         this.remoteStateStats = new RemotePersistenceStats();
+        this.indexMetadataUploadListeners = indexMetadataUploadListeners;
     }
 
     private BlobStoreTransferService getBlobStoreTransferService() {
@@ -233,10 +237,12 @@ public ClusterMetadataManifest writeFullMetadata(ClusterState clusterState, Stri
         // Write globalMetadata
         String globalMetadataFile = writeGlobalMetadata(clusterState);
 
+        List<IndexMetadata> toUpload = new ArrayList<>(clusterState.metadata().indices().values());
         // any validations before/after upload ?
         final List<UploadedIndexMetadata> allUploadedIndexMetadata = writeIndexMetadataParallel(
             clusterState,
-            new ArrayList<>(clusterState.metadata().indices().values())
+            toUpload,
+            Collections.emptyMap()
         );
         final ClusterMetadataManifest manifest = uploadManifest(
             clusterState,
@@ -301,9 +307,9 @@ public ClusterMetadataManifest writeIncrementalMetadata(
         }
 
         // Write Index Metadata
-        final Map<String, Long> previousStateIndexMetadataVersionByName = new HashMap<>();
+        final Map<String, IndexMetadata> previousStateIndexMetadataByName = new HashMap<>();
         for (final IndexMetadata indexMetadata : previousClusterState.metadata().indices().values()) {
-            previousStateIndexMetadataVersionByName.put(indexMetadata.getIndex().getName(), indexMetadata.getVersion());
+            previousStateIndexMetadataByName.put(indexMetadata.getIndex().getName(), indexMetadata);
         }
 
         int numIndicesUpdated = 0;
@@ -313,9 +319,12 @@ public ClusterMetadataManifest writeIncrementalMetadata(
             .collect(Collectors.toMap(UploadedIndexMetadata::getIndexName, Function.identity()));
 
         List<IndexMetadata> toUpload = new ArrayList<>();
-
+        // We prepare a map that contains the previous index metadata for the indexes for which version has changed.
+        Map<String, IndexMetadata> prevIndexMetadataByName = new HashMap<>();
         for (final IndexMetadata indexMetadata : clusterState.metadata().indices().values()) {
-            final Long previousVersion = previousStateIndexMetadataVersionByName.get(indexMetadata.getIndex().getName());
+            String indexName = indexMetadata.getIndex().getName();
+            final IndexMetadata prevIndexMetadata = previousStateIndexMetadataByName.get(indexName);
+            Long previousVersion = prevIndexMetadata != null ? prevIndexMetadata.getVersion() : null;
             if (previousVersion == null || indexMetadata.getVersion() != previousVersion) {
                 logger.debug(
                     "updating metadata for [{}], changing version from [{}] to [{}]",
@@ -325,18 +334,19 @@ public ClusterMetadataManifest writeIncrementalMetadata(
                 );
                 numIndicesUpdated++;
                 toUpload.add(indexMetadata);
+                prevIndexMetadataByName.put(indexName, prevIndexMetadata);
             } else {
                 numIndicesUnchanged++;
             }
-            previousStateIndexMetadataVersionByName.remove(indexMetadata.getIndex().getName());
+            previousStateIndexMetadataByName.remove(indexMetadata.getIndex().getName());
         }
 
-        List<UploadedIndexMetadata> uploadedIndexMetadataList = writeIndexMetadataParallel(clusterState, toUpload);
+        List<UploadedIndexMetadata> uploadedIndexMetadataList = writeIndexMetadataParallel(clusterState, toUpload, prevIndexMetadataByName);
         uploadedIndexMetadataList.forEach(
             uploadedIndexMetadata -> allUploadedIndexMetadata.put(uploadedIndexMetadata.getIndexName(), uploadedIndexMetadata)
         );
 
-        for (String removedIndexName : previousStateIndexMetadataVersionByName.keySet()) {
+        for (String removedIndexName : previousStateIndexMetadataByName.keySet()) {
             allUploadedIndexMetadata.remove(removedIndexName);
         }
         final ClusterMetadataManifest manifest = uploadManifest(
@@ -436,13 +446,18 @@ private String writeGlobalMetadata(ClusterState clusterState) throws IOException
      * Uploads provided IndexMetadata's to remote store in parallel. The call is blocking so the method waits for upload to finish and then return.
      *
      * @param clusterState current ClusterState
-     * @param toUpload list of IndexMetadata to upload
+     * @param toUpload     list of IndexMetadata to upload
      * @return {@code List<UploadedIndexMetadata>} list of IndexMetadata uploaded to remote
      */
-    private List<UploadedIndexMetadata> writeIndexMetadataParallel(ClusterState clusterState, List<IndexMetadata> toUpload)
-        throws IOException {
-        List<Exception> exceptionList = Collections.synchronizedList(new ArrayList<>(toUpload.size()));
-        final CountDownLatch latch = new CountDownLatch(toUpload.size());
+    private List<UploadedIndexMetadata> writeIndexMetadataParallel(
+        ClusterState clusterState,
+        List<IndexMetadata> toUpload,
+        Map<String, IndexMetadata> prevIndexMetadataByName
+    ) throws IOException {
+        assert Objects.nonNull(indexMetadataUploadListeners) : "indexMetadataUploadListeners can not be null";
+        int latchCount = toUpload.size() + indexMetadataUploadListeners.size();
+        List<Exception> exceptionList = Collections.synchronizedList(new ArrayList<>(latchCount));
+        final CountDownLatch latch = new CountDownLatch(latchCount);
         List<UploadedIndexMetadata> result = new ArrayList<>(toUpload.size());
 
         LatchedActionListener<UploadedIndexMetadata> latchedActionListener = new LatchedActionListener<>(
@@ -467,6 +482,8 @@ private List<UploadedIndexMetadata> writeIndexMetadataParallel(ClusterState clus
             writeIndexMetadataAsync(clusterState, indexMetadata, latchedActionListener);
         }
 
+        invokeIndexMetadataUploadListeners(toUpload, prevIndexMetadataByName, latch, exceptionList);
+
         try {
             if (latch.await(getIndexMetadataUploadTimeout().millis(), TimeUnit.MILLISECONDS) == false) {
                 RemoteStateTransferException ex = new RemoteStateTransferException(
@@ -506,11 +523,68 @@ private List<UploadedIndexMetadata> writeIndexMetadataParallel(ClusterState clus
         return result;
     }
 
+    /**
+     * Invokes the index metadata upload listener but does not wait for the execution to complete.
+     */
+    private void invokeIndexMetadataUploadListeners(
+        List<IndexMetadata> updatedIndexMetadataList,
+        Map<String, IndexMetadata> prevIndexMetadataByName,
+        CountDownLatch latch,
+        List<Exception> exceptionList
+    ) {
+        for (IndexMetadataUploadListener listener : indexMetadataUploadListeners) {
+            String listenerName = listener.getClass().getSimpleName();
+            listener.onUpload(
+                updatedIndexMetadataList,
+                prevIndexMetadataByName,
+                getIndexMetadataUploadActionListener(updatedIndexMetadataList, prevIndexMetadataByName, latch, exceptionList, listenerName)
+            );
+        }
+
+    }
+
+    private ActionListener<Void> getIndexMetadataUploadActionListener(
+        List<IndexMetadata> newIndexMetadataList,
+        Map<String, IndexMetadata> prevIndexMetadataByName,
+        CountDownLatch latch,
+        List<Exception> exceptionList,
+        String listenerName
+    ) {
+        long startTime = System.nanoTime();
+        return new LatchedActionListener<>(
+            ActionListener.wrap(
+                ignored -> logger.trace(
+                    new ParameterizedMessage(
+                        "listener={} : Invoked successfully with indexMetadataList={} prevIndexMetadataList={} tookTimeNs={}",
+                        listenerName,
+                        newIndexMetadataList,
+                        prevIndexMetadataByName.values(),
+                        (System.nanoTime() - startTime)
+                    )
+                ),
+                ex -> {
+                    logger.error(
+                        new ParameterizedMessage(
+                            "listener={} : Exception during invocation with indexMetadataList={} prevIndexMetadataList={} tookTimeNs={}",
+                            listenerName,
+                            newIndexMetadataList,
+                            prevIndexMetadataByName.values(),
+                            (System.nanoTime() - startTime)
+                        ),
+                        ex
+                    );
+                    exceptionList.add(ex);
+                }
+            ),
+            latch
+        );
+    }
+
     /**
      * Allows async Upload of IndexMetadata to remote
      *
-     * @param clusterState current ClusterState
-     * @param indexMetadata {@link IndexMetadata} to upload
+     * @param clusterState          current ClusterState
+     * @param indexMetadata         {@link IndexMetadata} to upload
      * @param latchedActionListener listener to respond back on after upload finishes
      */
     private void writeIndexMetadataAsync(
@@ -659,16 +733,6 @@ private void writeMetadataManifest(String clusterName, String clusterUUID, Clust
         );
     }
 
-    private String fetchPreviousClusterUUID(String clusterName, String clusterUUID) {
-        final Optional<ClusterMetadataManifest> latestManifest = getLatestClusterMetadataManifest(clusterName, clusterUUID);
-        if (!latestManifest.isPresent()) {
-            final String previousClusterUUID = getLastKnownUUIDFromRemote(clusterName);
-            assert !clusterUUID.equals(previousClusterUUID) : "Last cluster UUID is same current cluster UUID";
-            return previousClusterUUID;
-        }
-        return latestManifest.get().getPreviousClusterUUID();
-    }
-
     private BlobContainer indexMetadataContainer(String clusterName, String clusterUUID, String indexUUID) {
         // 123456789012_test-cluster/cluster-state/dsgYj10Nkso7/index/ftqsCnn9TgOX
         return blobStoreRepository.blobStore()
@@ -737,7 +801,7 @@ static String getManifestFileName(long term, long version, boolean committed) {
             (committed ? "C" : "P"), // C for committed and P for published
             RemoteStoreUtils.invertLong(System.currentTimeMillis()),
             String.valueOf(MANIFEST_CURRENT_CODEC_VERSION) // Keep the codec version at last place only, during read we reads last place to
-                                                           // determine codec version.
+            // determine codec version.
         );
     }
 
@@ -750,7 +814,7 @@ static String indexMetadataFileName(IndexMetadata indexMetadata) {
             RemoteStoreUtils.invertLong(indexMetadata.getVersion()),
             RemoteStoreUtils.invertLong(System.currentTimeMillis()),
             String.valueOf(INDEX_METADATA_CURRENT_CODEC_VERSION) // Keep the codec version at last place only, during read we reads last
-                                                                 // place to determine codec version.
+            // place to determine codec version.
         );
     }
 
@@ -772,8 +836,8 @@ private BlobPath getManifestFolderPath(String clusterName, String clusterUUID) {
     /**
      * Fetch latest index metadata from remote cluster state
      *
-     * @param clusterUUID uuid of cluster state to refer to in remote
-     * @param clusterName name of the cluster
+     * @param clusterUUID             uuid of cluster state to refer to in remote
+     * @param clusterName             name of the cluster
      * @param clusterMetadataManifest manifest file of cluster
      * @return {@code Map<String, IndexMetadata>} latest IndexUUID to IndexMetadata map
      */
@@ -795,8 +859,8 @@ private Map<String, IndexMetadata> getIndexMetadataMap(
     /**
      * Fetch index metadata from remote cluster state
      *
-     * @param clusterUUID uuid of cluster state to refer to in remote
-     * @param clusterName name of the cluster
+     * @param clusterUUID           uuid of cluster state to refer to in remote
+     * @param clusterName           name of the cluster
      * @param uploadedIndexMetadata {@link UploadedIndexMetadata} contains details about remote location of index metadata
      * @return {@link IndexMetadata}
      */
@@ -825,7 +889,6 @@ private IndexMetadata getIndexMetadata(String clusterName, String clusterUUID, U
      * @return {@link IndexMetadata}
      */
     public ClusterState getLatestClusterState(String clusterName, String clusterUUID) {
-        start();
         Optional<ClusterMetadataManifest> clusterMetadataManifest = getLatestClusterMetadataManifest(clusterName, clusterUUID);
         if (clusterMetadataManifest.isEmpty()) {
             throw new IllegalStateException(
@@ -989,6 +1052,7 @@ private List<String> createClusterChain(final Map<String, ClusterMetadataManifes
      * This method take a map of manifests for different cluster UUIDs and removes the
      * manifest of a cluster UUID if the latest metadata for that cluster UUID is equivalent
      * to the latest metadata of its previous UUID.
+     *
      * @return Trimmed map of manifests
      */
     private Map<String, ClusterMetadataManifest> trimClusterUUIDs(
@@ -1050,7 +1114,7 @@ private boolean isValidClusterUUID(ClusterMetadataManifest manifest) {
      *
      * @param clusterUUID uuid of cluster state to refer to in remote
      * @param clusterName name of the cluster
-     * @param limit max no of files to fetch
+     * @param limit       max no of files to fetch
      * @return all manifest file names
      */
     private List<BlobMetadata> getManifestFileNames(String clusterName, String clusterUUID, int limit) throws IllegalStateException {
@@ -1123,7 +1187,7 @@ private int getManifestCodecVersion(String fileName) {
         if (splitName.length == SPLITED_MANIFEST_FILE_LENGTH) {
             return Integer.parseInt(splitName[splitName.length - 1]); // Last value would be codec version.
         } else if (splitName.length < SPLITED_MANIFEST_FILE_LENGTH) { // Where codec is not part of file name, i.e. default codec version 0
-                                                                      // is used.
+            // is used.
             return ClusterMetadataManifest.CODEC_V0;
         } else {
             throw new IllegalArgumentException("Manifest file name is corrupted");
@@ -1141,7 +1205,7 @@ public void writeMetadataFailed() {
     /**
      * Exception for Remote state transfer.
      */
-    static class RemoteStateTransferException extends RuntimeException {
+    public static class RemoteStateTransferException extends RuntimeException {
 
         public RemoteStateTransferException(String errorDesc) {
             super(errorDesc);
@@ -1155,7 +1219,7 @@ public RemoteStateTransferException(String errorDesc, Throwable cause) {
     /**
      * Purges all remote cluster state against provided cluster UUIDs
      *
-     * @param clusterName name of the cluster
+     * @param clusterName  name of the cluster
      * @param clusterUUIDs clusteUUIDs for which the remote state needs to be purged
      */
     void deleteStaleUUIDsClusterMetadata(String clusterName, List<String> clusterUUIDs) {
@@ -1188,8 +1252,8 @@ public void onFailure(Exception e) {
     /**
      * Deletes older than last {@code versionsToRetain} manifests. Also cleans up unreferenced IndexMetadata associated with older manifests
      *
-     * @param clusterName name of the cluster
-     * @param clusterUUID uuid of cluster state to refer to in remote
+     * @param clusterName       name of the cluster
+     * @param clusterUUID       uuid of cluster state to refer to in remote
      * @param manifestsToRetain no of latest manifest files to keep in remote
      */
     // package private for testing
@@ -1308,7 +1372,8 @@ private void deleteStalePaths(String clusterName, String clusterUUID, List<Strin
 
     /**
      * Purges all remote cluster state against provided cluster UUIDs
-     * @param clusterState current state of the cluster
+     *
+     * @param clusterState      current state of the cluster
      * @param committedManifest last committed ClusterMetadataManifest
      */
     public void deleteStaleClusterUUIDs(ClusterState clusterState, ClusterMetadataManifest committedManifest) {
diff --git a/server/src/main/java/org/opensearch/http/HttpInfo.java b/server/src/main/java/org/opensearch/http/HttpInfo.java
index 10f2d50dacb14..4a39e40c471b1 100644
--- a/server/src/main/java/org/opensearch/http/HttpInfo.java
+++ b/server/src/main/java/org/opensearch/http/HttpInfo.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.http;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.logging.DeprecationLogger;
 import org.opensearch.common.network.InetAddresses;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -47,8 +48,9 @@
 /**
  * Information about an http connection
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class HttpInfo implements ReportingService.Info {
 
     private static final DeprecationLogger deprecationLogger = DeprecationLogger.getLogger(HttpInfo.class);
diff --git a/server/src/main/java/org/opensearch/http/HttpServerTransport.java b/server/src/main/java/org/opensearch/http/HttpServerTransport.java
index 890136cb67e60..012b69c29c1d4 100644
--- a/server/src/main/java/org/opensearch/http/HttpServerTransport.java
+++ b/server/src/main/java/org/opensearch/http/HttpServerTransport.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.http;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.lifecycle.LifecycleComponent;
 import org.opensearch.common.util.concurrent.ThreadContext;
 import org.opensearch.core.common.transport.BoundTransportAddress;
@@ -42,8 +43,9 @@
 /**
  * HTTP Transport server
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface HttpServerTransport extends LifecycleComponent, ReportingService<HttpInfo> {
 
     String HTTP_SERVER_WORKER_THREAD_NAME_PREFIX = "http_server_worker";
diff --git a/server/src/main/java/org/opensearch/http/HttpStats.java b/server/src/main/java/org/opensearch/http/HttpStats.java
index 078b84b7bc563..f69eff59e830d 100644
--- a/server/src/main/java/org/opensearch/http/HttpStats.java
+++ b/server/src/main/java/org/opensearch/http/HttpStats.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.http;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -43,8 +44,9 @@
 /**
  * Stats for HTTP connections
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class HttpStats implements Writeable, ToXContentFragment {
 
     private final long serverOpen;
diff --git a/server/src/main/java/org/opensearch/index/IndexModule.java b/server/src/main/java/org/opensearch/index/IndexModule.java
index 6ac10a221d49e..3c4cb4fd596c1 100644
--- a/server/src/main/java/org/opensearch/index/IndexModule.java
+++ b/server/src/main/java/org/opensearch/index/IndexModule.java
@@ -79,6 +79,7 @@
 import org.opensearch.index.store.remote.filecache.FileCache;
 import org.opensearch.index.translog.TranslogFactory;
 import org.opensearch.indices.IndicesQueryCache;
+import org.opensearch.indices.RemoteStoreSettings;
 import org.opensearch.indices.fielddata.cache.IndicesFieldDataCache;
 import org.opensearch.indices.mapper.MapperRegistry;
 import org.opensearch.indices.recovery.RecoverySettings;
@@ -604,8 +605,8 @@ public IndexService newIndexService(
         IndexStorePlugin.DirectoryFactory remoteDirectoryFactory,
         BiFunction<IndexSettings, ShardRouting, TranslogFactory> translogFactorySupplier,
         Supplier<TimeValue> clusterDefaultRefreshIntervalSupplier,
-        Supplier<TimeValue> clusterRemoteTranslogBufferIntervalSupplier,
-        RecoverySettings recoverySettings
+        RecoverySettings recoverySettings,
+        RemoteStoreSettings remoteStoreSettings
     ) throws IOException {
         final IndexEventListener eventListener = freeze();
         Function<IndexService, CheckedFunction<DirectoryReader, DirectoryReader, IOException>> readerWrapperFactory = indexReaderWrapper
@@ -663,8 +664,8 @@ public IndexService newIndexService(
                 recoveryStateFactory,
                 translogFactorySupplier,
                 clusterDefaultRefreshIntervalSupplier,
-                clusterRemoteTranslogBufferIntervalSupplier,
-                recoverySettings
+                recoverySettings,
+                remoteStoreSettings
             );
             success = true;
             return indexService;
diff --git a/server/src/main/java/org/opensearch/index/IndexService.java b/server/src/main/java/org/opensearch/index/IndexService.java
index 11dc4474cfa42..e501d7eff3f81 100644
--- a/server/src/main/java/org/opensearch/index/IndexService.java
+++ b/server/src/main/java/org/opensearch/index/IndexService.java
@@ -44,6 +44,7 @@
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
 import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.cluster.node.DiscoveryNodes;
 import org.opensearch.cluster.routing.ShardRouting;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.CheckedFunction;
@@ -94,6 +95,7 @@
 import org.opensearch.index.store.Store;
 import org.opensearch.index.translog.Translog;
 import org.opensearch.index.translog.TranslogFactory;
+import org.opensearch.indices.RemoteStoreSettings;
 import org.opensearch.indices.cluster.IndicesClusterStateService;
 import org.opensearch.indices.fielddata.cache.IndicesFieldDataCache;
 import org.opensearch.indices.mapper.MapperRegistry;
@@ -130,6 +132,7 @@
 import static java.util.Collections.emptyMap;
 import static java.util.Collections.unmodifiableMap;
 import static org.opensearch.common.collect.MapBuilder.newMapBuilder;
+import static org.opensearch.index.remote.RemoteMigrationIndexMetadataUpdater.indexHasRemoteStoreSettings;
 
 /**
  * The main OpenSearch index service
@@ -183,8 +186,8 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust
     private final ValuesSourceRegistry valuesSourceRegistry;
     private final BiFunction<IndexSettings, ShardRouting, TranslogFactory> translogFactorySupplier;
     private final Supplier<TimeValue> clusterDefaultRefreshIntervalSupplier;
-    private final Supplier<TimeValue> clusterRemoteTranslogBufferIntervalSupplier;
     private final RecoverySettings recoverySettings;
+    private final RemoteStoreSettings remoteStoreSettings;
 
     public IndexService(
         IndexSettings indexSettings,
@@ -219,8 +222,8 @@ public IndexService(
         IndexStorePlugin.RecoveryStateFactory recoveryStateFactory,
         BiFunction<IndexSettings, ShardRouting, TranslogFactory> translogFactorySupplier,
         Supplier<TimeValue> clusterDefaultRefreshIntervalSupplier,
-        Supplier<TimeValue> clusterRemoteTranslogBufferIntervalSupplier,
-        RecoverySettings recoverySettings
+        RecoverySettings recoverySettings,
+        RemoteStoreSettings remoteStoreSettings
     ) {
         super(indexSettings);
         this.allowExpensiveQueries = allowExpensiveQueries;
@@ -296,8 +299,8 @@ public IndexService(
         this.globalCheckpointTask = new AsyncGlobalCheckpointTask(this);
         this.retentionLeaseSyncTask = new AsyncRetentionLeaseSyncTask(this);
         this.translogFactorySupplier = translogFactorySupplier;
-        this.clusterRemoteTranslogBufferIntervalSupplier = clusterRemoteTranslogBufferIntervalSupplier;
         this.recoverySettings = recoverySettings;
+        this.remoteStoreSettings = remoteStoreSettings;
         updateFsyncTaskIfNecessary();
     }
 
@@ -461,7 +464,8 @@ public synchronized IndexShard createShard(
         final RemoteStoreStatsTrackerFactory remoteStoreStatsTrackerFactory,
         final RepositoriesService repositoriesService,
         final DiscoveryNode targetNode,
-        @Nullable DiscoveryNode sourceNode
+        @Nullable DiscoveryNode sourceNode,
+        DiscoveryNodes discoveryNodes
     ) throws IOException {
         Objects.requireNonNull(retentionLeaseSyncer);
         /*
@@ -497,20 +501,33 @@ public synchronized IndexShard createShard(
                 if (this.indexSettings.isRemoteStoreEnabled()) {
                     remoteDirectory = remoteDirectoryFactory.newDirectory(this.indexSettings, path);
                 } else {
-                    if (sourceNode != null && sourceNode.isRemoteStoreNode() == false) {
+                    if (sourceNode == null || sourceNode.isRemoteStoreNode() == false) {
                         if (routing.primary() == false) {
                             throw new IllegalStateException("Can't migrate a remote shard to replica before primary " + routing.shardId());
                         }
                         logger.info("DocRep shard {} is migrating to remote", shardId);
                         seedRemote = true;
                     }
+
                     remoteDirectory = ((RemoteSegmentStoreDirectoryFactory) remoteDirectoryFactory).newDirectory(
                         RemoteStoreNodeAttribute.getRemoteStoreSegmentRepo(this.indexSettings.getNodeSettings()),
                         this.indexSettings.getUUID(),
-                        shardId
+                        shardId,
+                        this.indexSettings.getRemoteStorePathStrategy()
                     );
                 }
                 remoteStore = new Store(shardId, this.indexSettings, remoteDirectory, lock, Store.OnClose.EMPTY, path);
+            } else {
+                // Disallow shards with remote store based settings to be created on non-remote store enabled nodes
+                // Even though we have `RemoteStoreMigrationAllocationDecider` in place to prevent something like this from happening at the
+                // allocation level,
+                // keeping this defensive check in place
+                // TODO: Remove this once remote to docrep migration is supported
+                if (indexHasRemoteStoreSettings(indexSettings)) {
+                    throw new IllegalStateException(
+                        "[{" + routing.shardId() + "}] Cannot initialize shards with remote store index settings on non-remote store nodes"
+                    );
+                }
             }
 
             Directory directory = directoryFactory.newDirectory(this.indexSettings, path);
@@ -548,10 +565,11 @@ public synchronized IndexShard createShard(
                 this.indexSettings.isSegRepEnabledOrRemoteNode() ? checkpointPublisher : null,
                 remoteStore,
                 remoteStoreStatsTrackerFactory,
-                clusterRemoteTranslogBufferIntervalSupplier,
                 nodeEnv.nodeId(),
                 recoverySettings,
-                seedRemote
+                remoteStoreSettings,
+                seedRemote,
+                discoveryNodes
             );
             eventListener.indexShardStateChanged(indexShard, null, indexShard.state(), "shard created");
             eventListener.afterIndexShardCreated(indexShard);
diff --git a/server/src/main/java/org/opensearch/index/IndexSettings.java b/server/src/main/java/org/opensearch/index/IndexSettings.java
index 7e49726c259cb..9d8ab6815eecc 100644
--- a/server/src/main/java/org/opensearch/index/IndexSettings.java
+++ b/server/src/main/java/org/opensearch/index/IndexSettings.java
@@ -48,6 +48,8 @@
 import org.opensearch.core.common.unit.ByteSizeUnit;
 import org.opensearch.core.common.unit.ByteSizeValue;
 import org.opensearch.core.index.Index;
+import org.opensearch.index.remote.RemoteStorePathStrategy;
+import org.opensearch.index.remote.RemoteStoreUtils;
 import org.opensearch.index.translog.Translog;
 import org.opensearch.indices.replication.common.ReplicationType;
 import org.opensearch.ingest.IngestService;
@@ -759,6 +761,8 @@ public static IndexMergePolicy fromString(String text) {
 
     private volatile String defaultSearchPipeline;
     private final boolean widenIndexSortType;
+    private final boolean assignedOnRemoteNode;
+    private final RemoteStorePathStrategy remoteStorePathStrategy;
 
     /**
      * The maximum age of a retention lease before it is considered expired.
@@ -982,6 +986,8 @@ public IndexSettings(final IndexMetadata indexMetadata, final Settings nodeSetti
          * Now this sortField (IndexSort) is stored in SegmentInfo and we need to maintain backward compatibility for them.
          */
         widenIndexSortType = IndexMetadata.SETTING_INDEX_VERSION_CREATED.get(settings).before(V_2_7_0);
+        assignedOnRemoteNode = RemoteStoreNodeAttribute.isRemoteDataAttributePresent(this.getNodeSettings());
+        remoteStorePathStrategy = RemoteStoreUtils.determineRemoteStorePathStrategy(indexMetadata);
 
         setEnableFuzzySetForDocId(scopedSettings.get(INDEX_DOC_ID_FUZZY_SET_ENABLED_SETTING));
         setDocIdFuzzySetFalsePositiveProbability(scopedSettings.get(INDEX_DOC_ID_FUZZY_SET_FALSE_POSITIVE_PROBABILITY_SETTING));
@@ -1227,15 +1233,11 @@ public int getNumberOfReplicas() {
      * proper index setting during the migration.
      */
     public boolean isSegRepEnabledOrRemoteNode() {
-        return ReplicationType.SEGMENT.equals(replicationType) || isRemoteNode();
+        return ReplicationType.SEGMENT.equals(replicationType) || isAssignedOnRemoteNode();
     }
 
     public boolean isSegRepLocalEnabled() {
-        return isSegRepEnabledOrRemoteNode() && !isRemoteStoreEnabled();
-    }
-
-    public boolean isSegRepWithRemoteEnabled() {
-        return isSegRepEnabledOrRemoteNode() && isRemoteStoreEnabled();
+        return ReplicationType.SEGMENT.equals(replicationType) && !isRemoteStoreEnabled();
     }
 
     /**
@@ -1245,8 +1247,8 @@ public boolean isRemoteStoreEnabled() {
         return isRemoteStoreEnabled;
     }
 
-    public boolean isRemoteNode() {
-        return RemoteStoreNodeAttribute.isRemoteDataAttributePresent(this.getNodeSettings());
+    public boolean isAssignedOnRemoteNode() {
+        return assignedOnRemoteNode;
     }
 
     /**
@@ -1905,4 +1907,8 @@ public double getDocIdFuzzySetFalsePositiveProbability() {
     public void setDocIdFuzzySetFalsePositiveProbability(double docIdFuzzySetFalsePositiveProbability) {
         this.docIdFuzzySetFalsePositiveProbability = docIdFuzzySetFalsePositiveProbability;
     }
+
+    public RemoteStorePathStrategy getRemoteStorePathStrategy() {
+        return remoteStorePathStrategy;
+    }
 }
diff --git a/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java b/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java
index bb35a09ccab46..c5c49f6dd5dd4 100644
--- a/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java
+++ b/server/src/main/java/org/opensearch/index/cache/request/ShardRequestCache.java
@@ -32,6 +32,8 @@
 
 package org.opensearch.index.cache.request;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.apache.lucene.util.Accountable;
 import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.metrics.CounterMetric;
@@ -45,13 +47,14 @@
 @PublicApi(since = "1.0.0")
 public final class ShardRequestCache {
 
+    private static final Logger logger = LogManager.getLogger(ShardRequestCache.class);
     final CounterMetric evictionsMetric = new CounterMetric();
     final CounterMetric totalMetric = new CounterMetric();
     final CounterMetric hitCount = new CounterMetric();
     final CounterMetric missCount = new CounterMetric();
 
     public RequestCacheStats stats() {
-        return new RequestCacheStats(totalMetric.count(), evictionsMetric.count(), hitCount.count(), missCount.count());
+        return new RequestCacheStats(Math.max(0, totalMetric.count()), evictionsMetric.count(), hitCount.count(), missCount.count());
     }
 
     public void onHit() {
@@ -62,21 +65,37 @@ public void onMiss() {
         missCount.inc();
     }
 
-    public void onCached(Accountable key, BytesReference value) {
-        totalMetric.inc(key.ramBytesUsed() + value.ramBytesUsed());
+    // Functions used to increment size by passing in the size directly, Used now, as we use ICacheKey<Key> in the IndicesRequestCache..
+    public void onCached(long keyRamBytesUsed, BytesReference value) {
+        totalMetric.inc(keyRamBytesUsed + value.ramBytesUsed());
     }
 
-    public void onRemoval(Accountable key, BytesReference value, boolean evicted) {
+    public void onRemoval(long keyRamBytesUsed, BytesReference value, boolean evicted) {
         if (evicted) {
             evictionsMetric.inc();
         }
-        long dec = 0;
-        if (key != null) {
-            dec += key.ramBytesUsed();
-        }
+        long dec = keyRamBytesUsed;
         if (value != null) {
             dec += value.ramBytesUsed();
         }
         totalMetric.dec(dec);
+        if (totalMetric.count() < 0) {
+            totalMetric.inc(dec);
+            logger.warn(
+                "Ignoring the operation to deduct memory: {} from RequestStats memory_size metric as it will "
+                    + "go negative. Current memory: {}. This is a bug.",
+                dec,
+                totalMetric.count()
+            );
+        }
+    }
+
+    // Old functions which increment size by passing in an Accountable. Functional but no longer used.
+    public void onCached(Accountable key, BytesReference value) {
+        totalMetric.inc(key.ramBytesUsed() + value.ramBytesUsed());
+    }
+
+    public void onRemoval(Accountable key, BytesReference value, boolean evicted) {
+        onRemoval(key.ramBytesUsed(), value, evicted);
     }
 }
diff --git a/server/src/main/java/org/opensearch/index/codec/fuzzy/LongArrayBackedBitSet.java b/server/src/main/java/org/opensearch/index/codec/fuzzy/LongArrayBackedBitSet.java
index bd4936aeec366..392a925c21143 100644
--- a/server/src/main/java/org/opensearch/index/codec/fuzzy/LongArrayBackedBitSet.java
+++ b/server/src/main/java/org/opensearch/index/codec/fuzzy/LongArrayBackedBitSet.java
@@ -39,7 +39,7 @@ class LongArrayBackedBitSet implements Accountable, Closeable {
     /**
      * Constructor which uses Lucene's IndexInput to read the bitset into a read-only buffer.
      * @param in IndexInput containing the serialized bitset.
-     * @throws IOException
+     * @throws IOException I/O exception
      */
     LongArrayBackedBitSet(IndexInput in) throws IOException {
         underlyingArrayLength = in.readLong();
diff --git a/server/src/main/java/org/opensearch/index/engine/EngineConfig.java b/server/src/main/java/org/opensearch/index/engine/EngineConfig.java
index 8106b65bddeec..fe2ce470cda03 100644
--- a/server/src/main/java/org/opensearch/index/engine/EngineConfig.java
+++ b/server/src/main/java/org/opensearch/index/engine/EngineConfig.java
@@ -236,6 +236,12 @@ private static void doValidateCodecSettings(final String codec) {
         Property.Dynamic
     );
 
+    public static final Setting<Boolean> INDEX_USE_COMPOUND_FILE = Setting.boolSetting(
+        "index.use_compound_file",
+        true,
+        Property.IndexScope
+    );
+
     private final TranslogConfig translogConfig;
 
     private final TranslogFactory translogFactory;
@@ -494,6 +500,10 @@ public boolean isReadOnlyReplica() {
         return indexSettings.isSegRepEnabledOrRemoteNode() && isReadOnlyReplica;
     }
 
+    public boolean useCompoundFile() {
+        return indexSettings.getValue(INDEX_USE_COMPOUND_FILE);
+    }
+
     /**
      * Returns the underlying startedPrimarySupplier.
      * @return the primary mode supplier.
diff --git a/server/src/main/java/org/opensearch/index/engine/InternalEngine.java b/server/src/main/java/org/opensearch/index/engine/InternalEngine.java
index 7bacec22fc850..59165b936aec8 100644
--- a/server/src/main/java/org/opensearch/index/engine/InternalEngine.java
+++ b/server/src/main/java/org/opensearch/index/engine/InternalEngine.java
@@ -2341,7 +2341,7 @@ private IndexWriterConfig getIndexWriterConfig() {
         iwc.setSimilarity(engineConfig.getSimilarity());
         iwc.setRAMBufferSizeMB(engineConfig.getIndexingBufferSize().getMbFrac());
         iwc.setCodec(engineConfig.getCodec());
-        iwc.setUseCompoundFile(true); // always use compound on flush - reduces # of file-handles on refresh
+        iwc.setUseCompoundFile(engineConfig.useCompoundFile());
         if (config().getIndexSort() != null) {
             iwc.setIndexSort(config().getIndexSort());
         }
diff --git a/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java b/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java
index 1e1825e1f8ace..d759423ce5a55 100644
--- a/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java
+++ b/server/src/main/java/org/opensearch/index/engine/NRTReplicationEngine.java
@@ -437,7 +437,7 @@ protected final void closeNoLock(String reason, CountDownLatch closedLatch) {
                  during promotion.
                  */
                 if (engineConfig.getIndexSettings().isRemoteStoreEnabled() == false
-                    && engineConfig.getIndexSettings().isRemoteNode() == false) {
+                    && engineConfig.getIndexSettings().isAssignedOnRemoteNode() == false) {
                     latestSegmentInfos.counter = latestSegmentInfos.counter + SI_COUNTER_INCREMENT;
                     latestSegmentInfos.changed();
                 }
diff --git a/server/src/main/java/org/opensearch/index/mapper/BooleanFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/BooleanFieldMapper.java
index 3c7925809415a..b4cf585c1329d 100644
--- a/server/src/main/java/org/opensearch/index/mapper/BooleanFieldMapper.java
+++ b/server/src/main/java/org/opensearch/index/mapper/BooleanFieldMapper.java
@@ -37,8 +37,11 @@
 import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.document.StoredField;
 import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BoostQuery;
+import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
-import org.apache.lucene.search.TermRangeQuery;
+import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.util.BytesRef;
 import org.opensearch.common.Booleans;
 import org.opensearch.common.Nullable;
@@ -55,8 +58,10 @@
 import java.time.ZoneId;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.function.Supplier;
 
 /**
@@ -175,6 +180,10 @@ public BooleanFieldType(String name, boolean searchable) {
             this(name, searchable, false, true, false, Collections.emptyMap());
         }
 
+        public BooleanFieldType(String name, boolean searchable, boolean hasDocValues) {
+            this(name, searchable, false, hasDocValues, false, Collections.emptyMap());
+        }
+
         @Override
         public String typeName() {
             return CONTENT_TYPE;
@@ -257,16 +266,81 @@ public DocValueFormat docValueFormat(@Nullable String format, ZoneId timeZone) {
             return DocValueFormat.BOOLEAN;
         }
 
+        @Override
+        public Query termQuery(Object value, QueryShardContext context) {
+            failIfNotIndexedAndNoDocValues();
+            if (!isSearchable()) {
+                return SortedNumericDocValuesField.newSlowExactQuery(name(), Values.TRUE.bytesEquals(indexedValueForSearch(value)) ? 1 : 0);
+            }
+            Query query = new TermQuery(new Term(name(), indexedValueForSearch(value)));
+            if (boost() != 1f) {
+                query = new BoostQuery(query, boost());
+            }
+            return query;
+        }
+
+        @Override
+        public Query termsQuery(List<?> values, QueryShardContext context) {
+            failIfNotIndexedAndNoDocValues();
+            int distinct = 0;
+            Set<?> distinctValues = new HashSet<>(values);
+            for (Object value : distinctValues) {
+                if (Values.TRUE.equals(indexedValueForSearch(value))) {
+                    distinct |= 2;
+                } else if (Values.FALSE.equals(indexedValueForSearch(value))) {
+                    distinct |= 1;
+                }
+                if (distinct == 3) {
+                    return this.existsQuery(context);
+                }
+            }
+            switch (distinct) {
+                case 1:
+                    return termQuery("false", context);
+                case 2:
+                    return termQuery("true", context);
+            }
+
+            return new MatchNoDocsQuery("Values did not contain True or False");
+        }
+
         @Override
         public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) {
-            failIfNotIndexed();
-            return new TermRangeQuery(
-                name(),
-                lowerTerm == null ? null : indexedValueForSearch(lowerTerm),
-                upperTerm == null ? null : indexedValueForSearch(upperTerm),
-                includeLower,
-                includeUpper
-            );
+            failIfNotIndexedAndNoDocValues();
+            if (lowerTerm == null) {
+                lowerTerm = false;
+                includeLower = true;
+
+            }
+            if (upperTerm == null) {
+                upperTerm = true;
+                includeUpper = true;
+
+            }
+
+            lowerTerm = indexedValueForSearch(lowerTerm);
+            upperTerm = indexedValueForSearch(upperTerm);
+
+            if (lowerTerm == upperTerm) {
+                if (!includeLower || !includeUpper) {
+                    return new MatchNoDocsQuery();
+                }
+                return termQuery(lowerTerm.equals(Values.TRUE), context);
+            }
+
+            if (lowerTerm.equals(Values.TRUE)) {
+                return new MatchNoDocsQuery();
+            }
+            if (!includeLower && !includeUpper) {
+                return new MatchNoDocsQuery();
+            } else if (!includeLower) {
+                return termQuery(true, context);
+            } else if (!includeUpper) {
+                return termQuery(false, context);
+            } else {
+                return this.existsQuery(context);
+            }
+
         }
     }
 
diff --git a/server/src/main/java/org/opensearch/index/mapper/ConstantKeywordFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/ConstantKeywordFieldMapper.java
new file mode 100644
index 0000000000000..f4730c70362d1
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/mapper/ConstantKeywordFieldMapper.java
@@ -0,0 +1,191 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.mapper;
+
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.Query;
+import org.opensearch.OpenSearchParseException;
+import org.opensearch.common.annotation.PublicApi;
+import org.opensearch.common.regex.Regex;
+import org.opensearch.index.fielddata.IndexFieldData;
+import org.opensearch.index.fielddata.plain.ConstantIndexFieldData;
+import org.opensearch.index.query.QueryShardContext;
+import org.opensearch.search.aggregations.support.CoreValuesSourceType;
+import org.opensearch.search.lookup.SearchLookup;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Supplier;
+
+/**
+ * Index specific field mapper
+ *
+ * @opensearch.api
+ */
+@PublicApi(since = "2.14.0")
+public class ConstantKeywordFieldMapper extends ParametrizedFieldMapper {
+
+    public static final String CONTENT_TYPE = "constant_keyword";
+
+    private static final String valuePropertyName = "value";
+
+    /**
+     * A {@link Mapper.TypeParser} for the constant keyword field.
+     *
+     * @opensearch.internal
+     */
+    public static class TypeParser implements Mapper.TypeParser {
+        @Override
+        public Mapper.Builder parse(String name, Map<String, Object> node, ParserContext parserContext) throws MapperParsingException {
+            if (!node.containsKey(valuePropertyName)) {
+                throw new OpenSearchParseException("Field [" + name + "] is missing required parameter [value]");
+            }
+            Object value = node.remove(valuePropertyName);
+            if (!(value instanceof String)) {
+                throw new OpenSearchParseException("Field [" + name + "] is expected to be a string value");
+            }
+            return new Builder(name, (String) value);
+        }
+    }
+
+    private static ConstantKeywordFieldMapper toType(FieldMapper in) {
+        return (ConstantKeywordFieldMapper) in;
+    }
+
+    /**
+     * Builder for the binary field mapper
+     *
+     * @opensearch.internal
+     */
+    public static class Builder extends ParametrizedFieldMapper.Builder {
+
+        private final Parameter<String> value;
+
+        public Builder(String name, String value) {
+            super(name);
+            this.value = Parameter.stringParam(valuePropertyName, false, m -> toType(m).value, value);
+        }
+
+        @Override
+        public List<Parameter<?>> getParameters() {
+            return Arrays.asList(value);
+        }
+
+        @Override
+        public ConstantKeywordFieldMapper build(BuilderContext context) {
+            return new ConstantKeywordFieldMapper(
+                name,
+                new ConstantKeywordFieldMapper.ConstantKeywordFieldType(buildFullName(context), value.getValue()),
+                multiFieldsBuilder.build(this, context),
+                copyTo.build(),
+                this
+            );
+        }
+    }
+
+    /**
+     * Field type for Index field mapper
+     *
+     * @opensearch.internal
+     */
+    @PublicApi(since = "2.14.0")
+    protected static final class ConstantKeywordFieldType extends ConstantFieldType {
+
+        protected final String value;
+
+        public ConstantKeywordFieldType(String name, String value) {
+            super(name, Collections.emptyMap());
+            this.value = value;
+        }
+
+        @Override
+        public String typeName() {
+            return CONTENT_TYPE;
+        }
+
+        @Override
+        protected boolean matches(String pattern, boolean caseInsensitive, QueryShardContext context) {
+            return Regex.simpleMatch(pattern, value, caseInsensitive);
+        }
+
+        @Override
+        public Query existsQuery(QueryShardContext context) {
+            return new MatchAllDocsQuery();
+        }
+
+        @Override
+        public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier<SearchLookup> searchLookup) {
+            return new ConstantIndexFieldData.Builder(fullyQualifiedIndexName, name(), CoreValuesSourceType.BYTES);
+        }
+
+        @Override
+        public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchLookup, String format) {
+            if (format != null) {
+                throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't " + "support formats.");
+            }
+
+            return new SourceValueFetcher(name(), context) {
+                @Override
+                protected Object parseSourceValue(Object value) {
+                    String keywordValue = value.toString();
+                    return Collections.singletonList(keywordValue);
+                }
+            };
+        }
+    }
+
+    private final String value;
+
+    protected ConstantKeywordFieldMapper(
+        String simpleName,
+        MappedFieldType mappedFieldType,
+        MultiFields multiFields,
+        CopyTo copyTo,
+        ConstantKeywordFieldMapper.Builder builder
+    ) {
+        super(simpleName, mappedFieldType, multiFields, copyTo);
+        this.value = builder.value.getValue();
+    }
+
+    public ParametrizedFieldMapper.Builder getMergeBuilder() {
+        return new ConstantKeywordFieldMapper.Builder(simpleName(), this.value).init(this);
+    }
+
+    @Override
+    protected void parseCreateField(ParseContext context) throws IOException {
+
+        final String value;
+        if (context.externalValueSet()) {
+            value = context.externalValue().toString();
+        } else {
+            value = context.parser().textOrNull();
+        }
+        if (value == null) {
+            throw new IllegalArgumentException("constant keyword field [" + name() + "] must have a value");
+        }
+
+        if (!value.equals(fieldType().value)) {
+            throw new IllegalArgumentException("constant keyword field [" + name() + "] must have a value of [" + this.value + "]");
+        }
+
+    }
+
+    @Override
+    public ConstantKeywordFieldMapper.ConstantKeywordFieldType fieldType() {
+        return (ConstantKeywordFieldMapper.ConstantKeywordFieldType) super.fieldType();
+    }
+
+    @Override
+    protected String contentType() {
+        return CONTENT_TYPE;
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java
index d98e6ea6af83d..b7ee3bb8ca3e3 100644
--- a/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java
+++ b/server/src/main/java/org/opensearch/index/mapper/DateFieldMapper.java
@@ -457,22 +457,30 @@ public Query rangeQuery(
             @Nullable DateMathParser forcedDateParser,
             QueryShardContext context
         ) {
-            failIfNotIndexed();
+            failIfNotIndexedAndNoDocValues();
             if (relation == ShapeRelation.DISJOINT) {
                 throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] does not support DISJOINT ranges");
             }
             DateMathParser parser = forcedDateParser == null ? dateMathParser : forcedDateParser;
             return dateRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, timeZone, parser, context, resolution, (l, u) -> {
-                Query query = LongPoint.newRangeQuery(name(), l, u);
-                if (hasDocValues()) {
+                if (isSearchable() && hasDocValues()) {
+                    Query query = LongPoint.newRangeQuery(name(), l, u);
                     Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u);
                     query = new IndexOrDocValuesQuery(query, dvQuery);
 
                     if (context.indexSortedOnField(name())) {
                         query = new IndexSortSortedNumericDocValuesRangeQuery(name(), l, u, query);
                     }
+                    return query;
+                }
+                if (hasDocValues()) {
+                    Query query = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u);
+                    if (context.indexSortedOnField(name())) {
+                        query = new IndexSortSortedNumericDocValuesRangeQuery(name(), l, u, query);
+                    }
+                    return query;
                 }
-                return query;
+                return LongPoint.newRangeQuery(name(), l, u);
             });
         }
 
@@ -543,6 +551,7 @@ public static long parseToLong(
 
         @Override
         public Query distanceFeatureQuery(Object origin, String pivot, float boost, QueryShardContext context) {
+            failIfNotIndexedAndNoDocValues();
             long originLong = parseToLong(origin, true, null, null, context::nowInMillis);
             TimeValue pivotTime = TimeValue.parseTimeValue(pivot, "distance_feature.pivot");
             return resolution.distanceFeatureQuery(name(), boost, originLong, pivotTime);
@@ -559,6 +568,10 @@ public Relation isFieldWithinQuery(
             DateMathParser dateParser,
             QueryRewriteContext context
         ) throws IOException {
+            // if we have only doc_values enabled we do not look at the BKD so we return an INTERSECTS by default
+            if (isSearchable() == false && hasDocValues()) {
+                return Relation.INTERSECTS;
+            }
             if (dateParser == null) {
                 dateParser = this.dateMathParser;
             }
diff --git a/server/src/main/java/org/opensearch/index/mapper/DerivedField.java b/server/src/main/java/org/opensearch/index/mapper/DerivedField.java
new file mode 100644
index 0000000000000..7ebe4e5f0b0e8
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/mapper/DerivedField.java
@@ -0,0 +1,90 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.mapper;
+
+import org.opensearch.common.annotation.PublicApi;
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.StreamOutput;
+import org.opensearch.core.common.io.stream.Writeable;
+import org.opensearch.core.xcontent.ToXContent;
+import org.opensearch.core.xcontent.ToXContentFragment;
+import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.script.Script;
+
+import java.io.IOException;
+import java.util.Objects;
+
+/**
+ * DerivedField representation: expects a name, type and script.
+ */
+@PublicApi(since = "2.14.0")
+public class DerivedField implements Writeable, ToXContentFragment {
+
+    private final String name;
+    private final String type;
+    private final Script script;
+
+    public DerivedField(String name, String type, Script script) {
+        this.name = name;
+        this.type = type;
+        this.script = script;
+    }
+
+    public DerivedField(StreamInput in) throws IOException {
+        name = in.readString();
+        type = in.readString();
+        script = new Script(in);
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(name);
+        out.writeString(type);
+        script.writeTo(out);
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params) throws IOException {
+        builder.startObject(name);
+        builder.field("type", type);
+        builder.field("script", script);
+        builder.endObject();
+        return builder;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public String getType() {
+        return type;
+    }
+
+    public Script getScript() {
+        return script;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(name, type, script);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (obj == null) {
+            return false;
+        }
+        if (getClass() != obj.getClass()) {
+            return false;
+        }
+        DerivedField other = (DerivedField) obj;
+        return Objects.equals(name, other.name) && Objects.equals(type, other.type) && Objects.equals(script, other.script);
+    }
+
+}
diff --git a/server/src/main/java/org/opensearch/index/mapper/DerivedFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/DerivedFieldMapper.java
new file mode 100644
index 0000000000000..c6ae71320c35c
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/mapper/DerivedFieldMapper.java
@@ -0,0 +1,158 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.mapper;
+
+import org.apache.lucene.index.IndexableField;
+import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.script.Script;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+
+/**
+ * A field mapper for derived fields
+ *
+ * @opensearch.internal
+ */
+public class DerivedFieldMapper extends ParametrizedFieldMapper {
+
+    public static final String CONTENT_TYPE = "derived";
+
+    private static DerivedFieldMapper toType(FieldMapper in) {
+        return (DerivedFieldMapper) in;
+    }
+
+    /**
+     * Builder for this field mapper
+     *
+     * @opensearch.internal
+     */
+    public static class Builder extends ParametrizedFieldMapper.Builder {
+        // TODO: The type of parameter may change here if the actual underlying FieldType object is needed
+        private final Parameter<String> type = Parameter.stringParam("type", false, m -> toType(m).type, "");
+
+        private final Parameter<Script> script = new Parameter<>(
+            "script",
+            false,
+            () -> null,
+            (n, c, o) -> o == null ? null : Script.parse(o),
+            m -> toType(m).script
+        ).setSerializerCheck((id, ic, value) -> value != null);
+
+        public Builder(String name) {
+            super(name);
+        }
+
+        public Builder(DerivedField derivedField) {
+            super(derivedField.getName());
+            this.type.setValue(derivedField.getType());
+            this.script.setValue(derivedField.getScript());
+        }
+
+        @Override
+        protected List<Parameter<?>> getParameters() {
+            return Arrays.asList(type, script);
+        }
+
+        @Override
+        public DerivedFieldMapper build(BuilderContext context) {
+            FieldMapper fieldMapper = DerivedFieldSupportedTypes.getFieldMapperFromType(type.getValue(), name, context);
+            Function<Object, IndexableField> fieldFunction = DerivedFieldSupportedTypes.getIndexableFieldGeneratorType(
+                type.getValue(),
+                name
+            );
+            DerivedFieldType ft = new DerivedFieldType(
+                new DerivedField(buildFullName(context), type.getValue(), script.getValue()),
+                fieldMapper,
+                fieldFunction
+            );
+            return new DerivedFieldMapper(name, ft, multiFieldsBuilder.build(this, context), copyTo.build(), this);
+        }
+    }
+
+    public static final TypeParser PARSER = new TypeParser((n, c) -> new Builder(n));
+    private final String type;
+    private final Script script;
+
+    protected DerivedFieldMapper(
+        String simpleName,
+        MappedFieldType mappedFieldType,
+        MultiFields multiFields,
+        CopyTo copyTo,
+        Builder builder
+    ) {
+        super(simpleName, mappedFieldType, multiFields, copyTo);
+        this.type = builder.type.getValue();
+        this.script = builder.script.getValue();
+    }
+
+    @Override
+    public DerivedFieldType fieldType() {
+        return (DerivedFieldType) super.fieldType();
+    }
+
+    @Override
+    protected void parseCreateField(ParseContext context) throws IOException {
+        // Leaving this empty as the parsing should be handled via the Builder when root object is parsed.
+        // The context would not contain anything in this case since the DerivedFieldMapper is not indexed or stored.
+        throw new UnsupportedOperationException("should not be invoked");
+    }
+
+    @Override
+    public ParametrizedFieldMapper.Builder getMergeBuilder() {
+        return new Builder(simpleName()).init(this);
+    }
+
+    @Override
+    protected String contentType() {
+        return CONTENT_TYPE;
+    }
+
+    @Override
+    protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException {
+        getMergeBuilder().toXContent(builder, includeDefaults);
+        multiFields.toXContent(builder, params);
+        copyTo.toXContent(builder, params);
+    }
+
+    public String getType() {
+        return type;
+    }
+
+    public Script getScript() {
+        return script;
+    }
+
+    public static Map<String, DerivedFieldType> getAllDerivedFieldTypeFromObject(
+        Map<String, Object> derivedFieldObject,
+        MapperService mapperService
+    ) {
+        Map<String, DerivedFieldType> derivedFieldTypes = new HashMap<>();
+        DocumentMapper documentMapper = mapperService.documentMapperParser().parse(DerivedFieldMapper.CONTENT_TYPE, derivedFieldObject);
+        if (documentMapper != null && documentMapper.mappers() != null) {
+            for (Mapper mapper : documentMapper.mappers()) {
+                if (mapper instanceof DerivedFieldMapper) {
+                    DerivedFieldType derivedFieldType = ((DerivedFieldMapper) mapper).fieldType();
+                    derivedFieldTypes.put(derivedFieldType.name(), derivedFieldType);
+                }
+            }
+        }
+        return derivedFieldTypes;
+    }
+
+    public static DerivedFieldType getDerivedFieldType(DerivedField derivedField, MapperService mapperService) {
+        BuilderContext builderContext = new Mapper.BuilderContext(mapperService.getIndexSettings().getSettings(), new ContentPath(1));
+        Builder builder = new Builder(derivedField);
+        return builder.build(builderContext).fieldType();
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/mapper/DerivedFieldSupportedTypes.java b/server/src/main/java/org/opensearch/index/mapper/DerivedFieldSupportedTypes.java
new file mode 100644
index 0000000000000..aa6936bf6529a
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/mapper/DerivedFieldSupportedTypes.java
@@ -0,0 +1,172 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.mapper;
+
+import org.apache.lucene.document.DoubleField;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.InetAddressPoint;
+import org.apache.lucene.document.KeywordField;
+import org.apache.lucene.document.LatLonPoint;
+import org.apache.lucene.document.LongField;
+import org.apache.lucene.document.LongPoint;
+import org.apache.lucene.index.IndexOptions;
+import org.apache.lucene.index.IndexableField;
+import org.opensearch.Version;
+import org.opensearch.common.Booleans;
+import org.opensearch.common.collect.Tuple;
+import org.opensearch.common.geo.GeoPoint;
+import org.opensearch.common.lucene.Lucene;
+import org.opensearch.common.network.InetAddresses;
+
+import java.net.InetAddress;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.function.BiFunction;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+/**
+ * Contains logic to get the FieldMapper for a given type of derived field. Also, for a given type of derived field,
+ * it is used to create an IndexableField for the provided type and object. It is useful when indexing into
+ * lucene MemoryIndex in {@link org.opensearch.index.query.DerivedFieldQuery}.
+ */
+public enum DerivedFieldSupportedTypes {
+
+    BOOLEAN("boolean", (name, context) -> {
+        BooleanFieldMapper.Builder builder = new BooleanFieldMapper.Builder(name);
+        return builder.build(context);
+    }, name -> o -> {
+        // Trying to mimic the logic for parsing source value as used in BooleanFieldMapper valueFetcher
+        Boolean value;
+        if (o instanceof Boolean) {
+            value = (Boolean) o;
+        } else {
+            String textValue = o.toString();
+            value = Booleans.parseBooleanStrict(textValue, false);
+        }
+        return new Field(name, value ? "T" : "F", BooleanFieldMapper.Defaults.FIELD_TYPE);
+    }, o -> o),
+    DATE("date", (name, context) -> {
+        // TODO: should we support mapping settings exposed by a given field type from derived fields too?
+        // for example, support `format` for date type?
+        DateFieldMapper.Builder builder = new DateFieldMapper.Builder(
+            name,
+            DateFieldMapper.Resolution.MILLISECONDS,
+            DateFieldMapper.getDefaultDateTimeFormatter(),
+            false,
+            Version.CURRENT
+        );
+        return builder.build(context);
+    }, name -> o -> new LongPoint(name, (long) o), o -> DateFieldMapper.getDefaultDateTimeFormatter().formatMillis((long) o)),
+    GEO_POINT("geo_point", (name, context) -> {
+        GeoPointFieldMapper.Builder builder = new GeoPointFieldMapper.Builder(name);
+        return builder.build(context);
+    }, name -> o -> {
+        // convert o to array of double
+        if (!(o instanceof Tuple) || !(((Tuple<?, ?>) o).v1() instanceof Double || !(((Tuple<?, ?>) o).v2() instanceof Double))) {
+            throw new ClassCastException("geo_point should be in format emit(double lat, double lon) for derived fields");
+        }
+        return new LatLonPoint(name, (double) ((Tuple<?, ?>) o).v1(), (double) ((Tuple<?, ?>) o).v2());
+    }, o -> new GeoPoint((double) ((Tuple) o).v1(), (double) ((Tuple) o).v2())),
+    IP("ip", (name, context) -> {
+        IpFieldMapper.Builder builder = new IpFieldMapper.Builder(name, false, Version.CURRENT);
+        return builder.build(context);
+    }, name -> o -> {
+        InetAddress address;
+        if (o instanceof InetAddress) {
+            address = (InetAddress) o;
+        } else {
+            address = InetAddresses.forString(o.toString());
+        }
+        return new InetAddressPoint(name, address);
+    }, o -> o),
+    KEYWORD("keyword", (name, context) -> {
+        FieldType dummyFieldType = new FieldType();
+        dummyFieldType.setIndexOptions(IndexOptions.DOCS_AND_FREQS);
+        KeywordFieldMapper.Builder keywordBuilder = new KeywordFieldMapper.Builder(name);
+        KeywordFieldMapper.KeywordFieldType keywordFieldType = keywordBuilder.buildFieldType(context, dummyFieldType);
+        keywordFieldType.setIndexAnalyzer(Lucene.KEYWORD_ANALYZER);
+        return new KeywordFieldMapper(
+            name,
+            dummyFieldType,
+            keywordFieldType,
+            keywordBuilder.multiFieldsBuilder.build(keywordBuilder, context),
+            keywordBuilder.copyTo.build(),
+            keywordBuilder
+        );
+    }, name -> o -> new KeywordField(name, (String) o, Field.Store.NO), o -> o),
+    LONG("long", (name, context) -> {
+        NumberFieldMapper.Builder longBuilder = new NumberFieldMapper.Builder(name, NumberFieldMapper.NumberType.LONG, false, false);
+        return longBuilder.build(context);
+    }, name -> o -> new LongField(name, Long.parseLong(o.toString()), Field.Store.NO), o -> o),
+    DOUBLE("double", (name, context) -> {
+        NumberFieldMapper.Builder doubleBuilder = new NumberFieldMapper.Builder(name, NumberFieldMapper.NumberType.DOUBLE, false, false);
+        return doubleBuilder.build(context);
+    }, name -> o -> new DoubleField(name, Double.parseDouble(o.toString()), Field.Store.NO), o -> o);
+
+    final String name;
+    private final BiFunction<String, Mapper.BuilderContext, FieldMapper> builder;
+
+    private final Function<String, Function<Object, IndexableField>> indexableFieldBuilder;
+
+    private final Function<Object, Object> valueForDisplay;
+
+    DerivedFieldSupportedTypes(
+        String name,
+        BiFunction<String, Mapper.BuilderContext, FieldMapper> builder,
+        Function<String, Function<Object, IndexableField>> indexableFieldBuilder,
+        Function<Object, Object> valueForDisplay
+    ) {
+        this.name = name;
+        this.builder = builder;
+        this.indexableFieldBuilder = indexableFieldBuilder;
+        this.valueForDisplay = valueForDisplay;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    private FieldMapper getFieldMapper(String name, Mapper.BuilderContext context) {
+        return builder.apply(name, context);
+    }
+
+    private Function<Object, IndexableField> getIndexableFieldGenerator(String name) {
+        return indexableFieldBuilder.apply(name);
+    }
+
+    private Function<Object, Object> getValueForDisplayGenerator() {
+        return valueForDisplay;
+    }
+
+    private static final Map<String, DerivedFieldSupportedTypes> enumMap = Arrays.stream(DerivedFieldSupportedTypes.values())
+        .collect(Collectors.toMap(DerivedFieldSupportedTypes::getName, enumValue -> enumValue));
+
+    public static FieldMapper getFieldMapperFromType(String type, String name, Mapper.BuilderContext context) {
+        if (!enumMap.containsKey(type)) {
+            throw new IllegalArgumentException("Type [" + type + "] isn't supported in Derived field context.");
+        }
+        return enumMap.get(type).getFieldMapper(name, context);
+    }
+
+    public static Function<Object, IndexableField> getIndexableFieldGeneratorType(String type, String name) {
+        if (!enumMap.containsKey(type)) {
+            throw new IllegalArgumentException("Type [" + type + "] isn't supported in Derived field context.");
+        }
+        return enumMap.get(type).getIndexableFieldGenerator(name);
+    }
+
+    public static Function<Object, Object> getValueForDisplayGenerator(String type) {
+        if (!enumMap.containsKey(type)) {
+            throw new IllegalArgumentException("Type [" + type + "] isn't supported in Derived field context.");
+        }
+        return enumMap.get(type).getValueForDisplayGenerator();
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/mapper/DerivedFieldType.java b/server/src/main/java/org/opensearch/index/mapper/DerivedFieldType.java
new file mode 100644
index 0000000000000..8b480819acd0e
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/mapper/DerivedFieldType.java
@@ -0,0 +1,290 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.mapper;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.queries.spans.SpanMultiTermQueryWrapper;
+import org.apache.lucene.queries.spans.SpanQuery;
+import org.apache.lucene.search.MultiTermQuery;
+import org.apache.lucene.search.Query;
+import org.opensearch.common.Nullable;
+import org.opensearch.common.geo.ShapeRelation;
+import org.opensearch.common.time.DateMathParser;
+import org.opensearch.common.unit.Fuzziness;
+import org.opensearch.geometry.Geometry;
+import org.opensearch.index.analysis.NamedAnalyzer;
+import org.opensearch.index.query.DerivedFieldQuery;
+import org.opensearch.index.query.QueryShardContext;
+import org.opensearch.script.DerivedFieldScript;
+import org.opensearch.search.lookup.SearchLookup;
+
+import java.io.IOException;
+import java.time.ZoneId;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+
+/**
+ * MappedFieldType for Derived Fields
+ * Contains logic to execute different type of queries on a derived field of given type.
+ * @opensearch.internal
+ */
+public final class DerivedFieldType extends MappedFieldType implements GeoShapeQueryable {
+
+    private final DerivedField derivedField;
+
+    FieldMapper typeFieldMapper;
+
+    final Function<Object, IndexableField> indexableFieldGenerator;
+
+    public DerivedFieldType(
+        DerivedField derivedField,
+        boolean isIndexed,
+        boolean isStored,
+        boolean hasDocValues,
+        Map<String, String> meta,
+        FieldMapper typeFieldMapper,
+        Function<Object, IndexableField> fieldFunction
+    ) {
+        super(derivedField.getName(), isIndexed, isStored, hasDocValues, typeFieldMapper.fieldType().getTextSearchInfo(), meta);
+        this.derivedField = derivedField;
+        this.typeFieldMapper = typeFieldMapper;
+        this.indexableFieldGenerator = fieldFunction;
+    }
+
+    public DerivedFieldType(DerivedField derivedField, FieldMapper typeFieldMapper, Function<Object, IndexableField> fieldFunction) {
+        this(derivedField, false, false, false, Collections.emptyMap(), typeFieldMapper, fieldFunction);
+    }
+
+    @Override
+    public String typeName() {
+        return "derived";
+    }
+
+    public String getType() {
+        return derivedField.getType();
+    }
+
+    public MappedFieldType getTypeMappedFieldType() {
+        return typeFieldMapper.mappedFieldType;
+    }
+
+    public NamedAnalyzer getIndexAnalyzer() {
+        return typeFieldMapper.mappedFieldType.indexAnalyzer();
+    }
+
+    @Override
+    public DerivedFieldValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchLookup, String format) {
+        if (format != null) {
+            throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
+        }
+        Function<Object, Object> valueForDisplay = DerivedFieldSupportedTypes.getValueForDisplayGenerator(getType());
+        return new DerivedFieldValueFetcher(
+            getDerivedFieldLeafFactory(context, searchLookup == null ? context.lookup() : searchLookup),
+            valueForDisplay,
+            indexableFieldGenerator
+        );
+    }
+
+    @Override
+    public Query termQuery(Object value, QueryShardContext context) {
+        Query query = typeFieldMapper.mappedFieldType.termQuery(value, context);
+        DerivedFieldValueFetcher valueFetcher = valueFetcher(context, context.lookup(), null);
+        return new DerivedFieldQuery(query, valueFetcher, context.lookup(), getIndexAnalyzer());
+    }
+
+    @Override
+    public Query termQueryCaseInsensitive(Object value, @Nullable QueryShardContext context) {
+        Query query = typeFieldMapper.mappedFieldType.termQueryCaseInsensitive(value, context);
+        DerivedFieldValueFetcher valueFetcher = valueFetcher(context, context.lookup(), null);
+        return new DerivedFieldQuery(query, valueFetcher, context.lookup(), getIndexAnalyzer());
+    }
+
+    @Override
+    public Query termsQuery(List<?> values, @Nullable QueryShardContext context) {
+        Query query = typeFieldMapper.mappedFieldType.termsQuery(values, context);
+        DerivedFieldValueFetcher valueFetcher = valueFetcher(context, context.lookup(), null);
+        return new DerivedFieldQuery(query, valueFetcher, context.lookup(), getIndexAnalyzer());
+    }
+
+    @Override
+    public Query rangeQuery(
+        Object lowerTerm,
+        Object upperTerm,
+        boolean includeLower,
+        boolean includeUpper,
+        ShapeRelation relation,
+        ZoneId timeZone,
+        DateMathParser parser,
+        QueryShardContext context
+    ) {
+        Query query = typeFieldMapper.mappedFieldType.rangeQuery(
+            lowerTerm,
+            upperTerm,
+            includeLower,
+            includeUpper,
+            relation,
+            timeZone,
+            parser,
+            context
+        );
+        DerivedFieldValueFetcher valueFetcher = valueFetcher(context, context.lookup(), null);
+        return new DerivedFieldQuery(query, valueFetcher, context.lookup(), getIndexAnalyzer());
+    }
+
+    @Override
+    public Query fuzzyQuery(
+        Object value,
+        Fuzziness fuzziness,
+        int prefixLength,
+        int maxExpansions,
+        boolean transpositions,
+        QueryShardContext context
+    ) {
+        Query query = typeFieldMapper.mappedFieldType.fuzzyQuery(value, fuzziness, prefixLength, maxExpansions, transpositions, context);
+        DerivedFieldValueFetcher valueFetcher = valueFetcher(context, context.lookup(), null);
+        return new DerivedFieldQuery(query, valueFetcher, context.lookup(), getIndexAnalyzer());
+    }
+
+    @Override
+    public Query fuzzyQuery(
+        Object value,
+        Fuzziness fuzziness,
+        int prefixLength,
+        int maxExpansions,
+        boolean transpositions,
+        @Nullable MultiTermQuery.RewriteMethod method,
+        QueryShardContext context
+    ) {
+        Query query = typeFieldMapper.mappedFieldType.fuzzyQuery(
+            value,
+            fuzziness,
+            prefixLength,
+            maxExpansions,
+            transpositions,
+            method,
+            context
+        );
+        DerivedFieldValueFetcher valueFetcher = valueFetcher(context, context.lookup(), null);
+        return new DerivedFieldQuery(query, valueFetcher, context.lookup(), getIndexAnalyzer());
+    }
+
+    @Override
+    public Query prefixQuery(
+        String value,
+        @Nullable MultiTermQuery.RewriteMethod method,
+        boolean caseInsensitive,
+        QueryShardContext context
+    ) {
+        Query query = typeFieldMapper.mappedFieldType.prefixQuery(value, method, caseInsensitive, context);
+        DerivedFieldValueFetcher valueFetcher = valueFetcher(context, context.lookup(), null);
+        return new DerivedFieldQuery(query, valueFetcher, context.lookup(), getIndexAnalyzer());
+    }
+
+    @Override
+    public Query wildcardQuery(
+        String value,
+        @Nullable MultiTermQuery.RewriteMethod method,
+        boolean caseInsensitive,
+        QueryShardContext context
+    ) {
+        Query query = typeFieldMapper.mappedFieldType.wildcardQuery(value, method, caseInsensitive, context);
+        DerivedFieldValueFetcher valueFetcher = valueFetcher(context, context.lookup(), null);
+        return new DerivedFieldQuery(query, valueFetcher, context.lookup(), getIndexAnalyzer());
+    }
+
+    @Override
+    public Query normalizedWildcardQuery(String value, @Nullable MultiTermQuery.RewriteMethod method, QueryShardContext context) {
+        Query query = typeFieldMapper.mappedFieldType.normalizedWildcardQuery(value, method, context);
+        DerivedFieldValueFetcher valueFetcher = valueFetcher(context, context.lookup(), null);
+        return new DerivedFieldQuery(query, valueFetcher, context.lookup(), getIndexAnalyzer());
+    }
+
+    @Override
+    public Query regexpQuery(
+        String value,
+        int syntaxFlags,
+        int matchFlags,
+        int maxDeterminizedStates,
+        @Nullable MultiTermQuery.RewriteMethod method,
+        QueryShardContext context
+    ) {
+        Query query = typeFieldMapper.mappedFieldType.regexpQuery(value, syntaxFlags, matchFlags, maxDeterminizedStates, method, context);
+        DerivedFieldValueFetcher valueFetcher = valueFetcher(context, context.lookup(), null);
+        return new DerivedFieldQuery(query, valueFetcher, context.lookup(), getIndexAnalyzer());
+    }
+
+    @Override
+    public Query phraseQuery(TokenStream stream, int slop, boolean enablePositionIncrements, QueryShardContext context) throws IOException {
+        Query query = typeFieldMapper.mappedFieldType.phraseQuery(stream, slop, enablePositionIncrements, context);
+        DerivedFieldValueFetcher valueFetcher = valueFetcher(context, context.lookup(), null);
+        return new DerivedFieldQuery(query, valueFetcher, context.lookup(), getIndexAnalyzer());
+    }
+
+    @Override
+    public Query multiPhraseQuery(TokenStream stream, int slop, boolean enablePositionIncrements, QueryShardContext context)
+        throws IOException {
+        Query query = typeFieldMapper.mappedFieldType.multiPhraseQuery(stream, slop, enablePositionIncrements, context);
+        DerivedFieldValueFetcher valueFetcher = valueFetcher(context, context.lookup(), null);
+        return new DerivedFieldQuery(query, valueFetcher, context.lookup(), getIndexAnalyzer());
+    }
+
+    @Override
+    public Query phrasePrefixQuery(TokenStream stream, int slop, int maxExpansions, QueryShardContext context) throws IOException {
+        Query query = typeFieldMapper.mappedFieldType.phrasePrefixQuery(stream, slop, maxExpansions, context);
+        DerivedFieldValueFetcher valueFetcher = valueFetcher(context, context.lookup(), null);
+        return new DerivedFieldQuery(query, valueFetcher, context.lookup(), getIndexAnalyzer());
+    }
+
+    @Override
+    public SpanQuery spanPrefixQuery(String value, SpanMultiTermQueryWrapper.SpanRewriteMethod method, QueryShardContext context) {
+        throw new IllegalArgumentException(
+            "Can only use span prefix queries on text fields - not on [" + name() + "] which is of type [" + typeName() + "]"
+        );
+    }
+
+    @Override
+    public Query distanceFeatureQuery(Object origin, String pivot, float boost, QueryShardContext context) {
+        Query query = typeFieldMapper.mappedFieldType.distanceFeatureQuery(origin, pivot, boost, context);
+        DerivedFieldValueFetcher valueFetcher = valueFetcher(context, context.lookup(), null);
+        return new DerivedFieldQuery(query, valueFetcher, context.lookup(), getIndexAnalyzer());
+    }
+
+    @Override
+    public Query geoShapeQuery(Geometry shape, String fieldName, ShapeRelation relation, QueryShardContext context) {
+        Query query = ((GeoShapeQueryable) (typeFieldMapper.mappedFieldType)).geoShapeQuery(shape, fieldName, relation, context);
+        DerivedFieldValueFetcher valueFetcher = valueFetcher(context, context.lookup(), null);
+        return new DerivedFieldQuery(query, valueFetcher, context.lookup(), getIndexAnalyzer());
+    }
+
+    @Override
+    public Query existsQuery(QueryShardContext context) {
+        throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] does not support exist queries");
+    }
+
+    @Override
+    public boolean isAggregatable() {
+        return false;
+    }
+
+    private DerivedFieldScript.LeafFactory getDerivedFieldLeafFactory(QueryShardContext context, SearchLookup searchLookup) {
+        if (!context.documentMapper("").sourceMapper().enabled()) {
+            throw new IllegalArgumentException(
+                "DerivedFieldQuery error: unable to fetch fields from _source field: _source is disabled in the mappings "
+                    + "for index ["
+                    + context.index().getName()
+                    + "]"
+            );
+        }
+        DerivedFieldScript.Factory factory = context.compile(derivedField.getScript(), DerivedFieldScript.CONTEXT);
+        return factory.newFactory(derivedField.getScript().getParams(), searchLookup);
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/mapper/DerivedFieldValueFetcher.java b/server/src/main/java/org/opensearch/index/mapper/DerivedFieldValueFetcher.java
new file mode 100644
index 0000000000000..2d9379e04c512
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/mapper/DerivedFieldValueFetcher.java
@@ -0,0 +1,81 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.mapper;
+
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.LeafReaderContext;
+import org.opensearch.common.annotation.PublicApi;
+import org.opensearch.script.DerivedFieldScript;
+import org.opensearch.search.lookup.SourceLookup;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Function;
+
+/**
+ * The value fetcher contains logic to execute script and fetch the value in form of list of object.
+ * It expects DerivedFieldScript.LeafFactory as an input and sets the contract with consumer to call
+ * {@link #setNextReader(LeafReaderContext)} whenever a segment is switched.
+ */
+@PublicApi(since = "2.14.0")
+public final class DerivedFieldValueFetcher implements ValueFetcher {
+    private DerivedFieldScript derivedFieldScript;
+    private final DerivedFieldScript.LeafFactory derivedFieldScriptFactory;
+
+    private final Function<Object, Object> valueForDisplay;
+    private final Function<Object, IndexableField> indexableFieldFunction;
+
+    public DerivedFieldValueFetcher(
+        DerivedFieldScript.LeafFactory derivedFieldScriptFactory,
+        Function<Object, Object> valueForDisplay,
+        Function<Object, IndexableField> indexableFieldFunction
+    ) {
+        this.derivedFieldScriptFactory = derivedFieldScriptFactory;
+        this.valueForDisplay = valueForDisplay;
+        this.indexableFieldFunction = indexableFieldFunction;
+    }
+
+    @Override
+    public List<Object> fetchValues(SourceLookup lookup) {
+        List<Object> values = fetchValuesInternal(lookup);
+        if (values.isEmpty()) {
+            return values;
+        }
+        List<Object> result = new ArrayList<>();
+        for (Object v : values) {
+            result.add(valueForDisplay.apply(v));
+        }
+        return result;
+    }
+
+    private List<Object> fetchValuesInternal(SourceLookup lookup) {
+        derivedFieldScript.setDocument(lookup.docId());
+        derivedFieldScript.execute();
+        return derivedFieldScript.getEmittedValues();
+    }
+
+    public List<IndexableField> getIndexableField(SourceLookup lookup) {
+        List<Object> values = fetchValuesInternal(lookup);
+        List<IndexableField> indexableFields = new ArrayList<>();
+        for (Object v : values) {
+            indexableFields.add(indexableFieldFunction.apply(v));
+        }
+        return indexableFields;
+    }
+
+    @Override
+    public void setNextReader(LeafReaderContext context) {
+        try {
+            derivedFieldScript = derivedFieldScriptFactory.newInstance(context);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/mapper/DocumentMapperParser.java b/server/src/main/java/org/opensearch/index/mapper/DocumentMapperParser.java
index fa736f84895eb..7c69b679e5a55 100644
--- a/server/src/main/java/org/opensearch/index/mapper/DocumentMapperParser.java
+++ b/server/src/main/java/org/opensearch/index/mapper/DocumentMapperParser.java
@@ -133,7 +133,7 @@ public DocumentMapper parse(@Nullable String type, CompressedXContent source) th
     }
 
     @SuppressWarnings({ "unchecked" })
-    private DocumentMapper parse(String type, Map<String, Object> mapping) throws MapperParsingException {
+    public DocumentMapper parse(String type, Map<String, Object> mapping) throws MapperParsingException {
         if (type == null) {
             throw new MapperParsingException("Failed to derive type");
         }
diff --git a/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java
index c14b2c92c89c3..42b974734e5e7 100644
--- a/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java
+++ b/server/src/main/java/org/opensearch/index/mapper/KeywordFieldMapper.java
@@ -218,7 +218,7 @@ protected List<Parameter<?>> getParameters() {
             );
         }
 
-        private KeywordFieldType buildFieldType(BuilderContext context, FieldType fieldType) {
+        protected KeywordFieldType buildFieldType(BuilderContext context, FieldType fieldType) {
             NamedAnalyzer normalizer = Lucene.KEYWORD_ANALYZER;
             NamedAnalyzer searchAnalyzer = Lucene.KEYWORD_ANALYZER;
             String normalizerName = this.normalizer.getValue();
diff --git a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java
index 39b0ec224b57c..92ffdb60e6cde 100644
--- a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java
+++ b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java
@@ -291,6 +291,15 @@ protected static boolean parseObjectOrDocumentTypeProperties(
             } else if (fieldName.equals("enabled")) {
                 builder.enabled(XContentMapValues.nodeBooleanValue(fieldNode, fieldName + ".enabled"));
                 return true;
+            } else if (fieldName.equals("derived")) {
+                if (fieldNode instanceof Collection && ((Collection) fieldNode).isEmpty()) {
+                    // nothing to do here, empty (to support "derived: []" case)
+                } else if (fieldNode instanceof Map) {
+                    parseDerived(builder, (Map<String, Object>) fieldNode, parserContext);
+                } else {
+                    throw new OpenSearchParseException("derived must be a map type");
+                }
+                return true;
             } else if (fieldName.equals("properties")) {
                 if (fieldNode instanceof Collection && ((Collection) fieldNode).isEmpty()) {
                     // nothing to do here, empty (to support "properties: []" case)
@@ -349,6 +358,55 @@ protected static void parseNested(
             }
         }
 
+        protected static void parseDerived(ObjectMapper.Builder objBuilder, Map<String, Object> derivedNode, ParserContext parserContext) {
+            Iterator<Map.Entry<String, Object>> iterator = derivedNode.entrySet().iterator();
+            while (iterator.hasNext()) {
+                Map.Entry<String, Object> entry = iterator.next();
+                String fieldName = entry.getKey();
+                // Should accept empty arrays, as a work around for when the
+                // user can't provide an empty Map. (PHP for example)
+                boolean isEmptyList = entry.getValue() instanceof List && ((List<?>) entry.getValue()).isEmpty();
+
+                if (entry.getValue() instanceof Map) {
+                    @SuppressWarnings("unchecked")
+                    Map<String, Object> node = (Map<String, Object>) entry.getValue();
+
+                    // Derived fields are a bit unique in that the 'type' attribute does not map to the TypeParser
+                    // like it would for traditional fields in properties.
+                    // So in this case, the DerivedFieldMapper's TypeParser will explicitly be used
+                    Mapper.TypeParser typeParser = parserContext.typeParser(DerivedFieldMapper.CONTENT_TYPE);
+                    String[] fieldNameParts = fieldName.split("\\.");
+                    // field name is just ".", which is invalid
+                    if (fieldNameParts.length < 1) {
+                        throw new MapperParsingException("Invalid field name " + fieldName);
+                    }
+                    String realFieldName = fieldNameParts[fieldNameParts.length - 1];
+                    Mapper.Builder<?> fieldBuilder = typeParser.parse(realFieldName, node, parserContext);
+                    for (int i = fieldNameParts.length - 2; i >= 0; --i) {
+                        ObjectMapper.Builder<?> intermediate = new ObjectMapper.Builder<>(fieldNameParts[i]);
+                        intermediate.add(fieldBuilder);
+                        fieldBuilder = intermediate;
+                    }
+                    objBuilder.add(fieldBuilder);
+                    node.remove("type");
+                    DocumentMapperParser.checkNoRemainingFields(fieldName, node, parserContext.indexVersionCreated());
+                    iterator.remove();
+                } else if (isEmptyList) {
+                    iterator.remove();
+                } else {
+                    throw new MapperParsingException(
+                        "Expected map for property [derived_fields] on field [" + fieldName + "] but got a " + fieldName.getClass()
+                    );
+                }
+            }
+
+            DocumentMapperParser.checkNoRemainingFields(
+                derivedNode,
+                parserContext.indexVersionCreated(),
+                "DocType mapping definition has unsupported parameters: "
+            );
+        }
+
         protected static void parseProperties(ObjectMapper.Builder objBuilder, Map<String, Object> propsNode, ParserContext parserContext) {
             Iterator<Map.Entry<String, Object>> iterator = propsNode.entrySet().iterator();
             while (iterator.hasNext()) {
@@ -663,7 +721,21 @@ public void toXContent(XContentBuilder builder, Params params, ToXContent custom
         doXContent(builder, params);
 
         // sort the mappers so we get consistent serialization format
-        Mapper[] sortedMappers = mappers.values().stream().toArray(size -> new Mapper[size]);
+        Mapper[] derivedSortedMappers = mappers.values()
+            .stream()
+            .filter(m -> m instanceof DerivedFieldMapper)
+            .toArray(size -> new Mapper[size]);
+        Arrays.sort(derivedSortedMappers, new Comparator<Mapper>() {
+            @Override
+            public int compare(Mapper o1, Mapper o2) {
+                return o1.name().compareTo(o2.name());
+            }
+        });
+
+        Mapper[] sortedMappers = mappers.values()
+            .stream()
+            .filter(m -> !(m instanceof DerivedFieldMapper))
+            .toArray(size -> new Mapper[size]);
         Arrays.sort(sortedMappers, new Comparator<Mapper>() {
             @Override
             public int compare(Mapper o1, Mapper o2) {
@@ -672,6 +744,17 @@ public int compare(Mapper o1, Mapper o2) {
         });
 
         int count = 0;
+        for (Mapper mapper : derivedSortedMappers) {
+            if (count++ == 0) {
+                builder.startObject("derived");
+            }
+            mapper.toXContent(builder, params);
+        }
+        if (count > 0) {
+            builder.endObject();
+        }
+
+        count = 0;
         for (Mapper mapper : sortedMappers) {
             if (!(mapper instanceof MetadataFieldMapper)) {
                 if (count++ == 0) {
diff --git a/server/src/main/java/org/opensearch/index/mapper/ParametrizedFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/ParametrizedFieldMapper.java
index f4723b6178137..4f4ec45001a54 100644
--- a/server/src/main/java/org/opensearch/index/mapper/ParametrizedFieldMapper.java
+++ b/server/src/main/java/org/opensearch/index/mapper/ParametrizedFieldMapper.java
@@ -670,7 +670,11 @@ public final void parse(String name, ParserContext parserContext, Map<String, Ob
                     deprecatedParamsMap.put(deprecatedName, param);
                 }
             }
-            String type = (String) fieldNode.remove("type");
+            String type = (String) fieldNode.get("type");
+            if (paramsMap.get("type") == null) {
+                fieldNode.remove("type");
+            }
+
             for (Iterator<Map.Entry<String, Object>> iterator = fieldNode.entrySet().iterator(); iterator.hasNext();) {
                 Map.Entry<String, Object> entry = iterator.next();
                 final String propName = entry.getKey();
diff --git a/server/src/main/java/org/opensearch/index/query/DerivedFieldQuery.java b/server/src/main/java/org/opensearch/index/query/DerivedFieldQuery.java
new file mode 100644
index 0000000000000..42ac61bf98f73
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/query/DerivedFieldQuery.java
@@ -0,0 +1,131 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.query;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.memory.MemoryIndex;
+import org.apache.lucene.search.ConstantScoreScorer;
+import org.apache.lucene.search.ConstantScoreWeight;
+import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.QueryVisitor;
+import org.apache.lucene.search.ScoreMode;
+import org.apache.lucene.search.Scorer;
+import org.apache.lucene.search.TwoPhaseIterator;
+import org.apache.lucene.search.Weight;
+import org.opensearch.index.mapper.DerivedFieldValueFetcher;
+import org.opensearch.search.lookup.LeafSearchLookup;
+import org.opensearch.search.lookup.SearchLookup;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * DerivedFieldQuery used for querying derived fields. It contains the logic to execute an input lucene query against
+ * DerivedField. It also accepts DerivedFieldValueFetcher and SearchLookup as an input.
+ */
+public final class DerivedFieldQuery extends Query {
+    private final Query query;
+    private final DerivedFieldValueFetcher valueFetcher;
+    private final SearchLookup searchLookup;
+    private final Analyzer indexAnalyzer;
+
+    /**
+     * @param query lucene query to be executed against the derived field
+     * @param valueFetcher DerivedFieldValueFetcher ValueFetcher to fetch the value of a derived field from _source
+     *                     using LeafSearchLookup
+     * @param searchLookup SearchLookup to get the LeafSearchLookup look used by valueFetcher to fetch the _source
+     */
+    public DerivedFieldQuery(Query query, DerivedFieldValueFetcher valueFetcher, SearchLookup searchLookup, Analyzer indexAnalyzer) {
+        this.query = query;
+        this.valueFetcher = valueFetcher;
+        this.searchLookup = searchLookup;
+        this.indexAnalyzer = indexAnalyzer;
+    }
+
+    @Override
+    public void visit(QueryVisitor visitor) {
+        query.visit(visitor);
+    }
+
+    @Override
+    public Query rewrite(IndexSearcher indexSearcher) throws IOException {
+        Query rewritten = indexSearcher.rewrite(query);
+        if (rewritten == query) {
+            return this;
+        }
+        return new DerivedFieldQuery(rewritten, valueFetcher, searchLookup, indexAnalyzer);
+    }
+
+    @Override
+    public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
+
+        return new ConstantScoreWeight(this, boost) {
+            @Override
+            public Scorer scorer(LeafReaderContext context) {
+                DocIdSetIterator approximation = DocIdSetIterator.all(context.reader().maxDoc());
+                valueFetcher.setNextReader(context);
+                LeafSearchLookup leafSearchLookup = searchLookup.getLeafSearchLookup(context);
+                TwoPhaseIterator twoPhase = new TwoPhaseIterator(approximation) {
+                    @Override
+                    public boolean matches() {
+                        leafSearchLookup.source().setSegmentAndDocument(context, approximation.docID());
+                        List<IndexableField> indexableFields = valueFetcher.getIndexableField(leafSearchLookup.source());
+                        // TODO: in case of errors from script, should it be ignored and treated as missing field
+                        // by using a configurable setting?
+                        MemoryIndex memoryIndex = new MemoryIndex();
+                        for (IndexableField indexableField : indexableFields) {
+                            memoryIndex.addField(indexableField, indexAnalyzer);
+                        }
+                        float score = memoryIndex.search(query);
+                        return score > 0.0f;
+                    }
+
+                    @Override
+                    public float matchCost() {
+                        // TODO: how can we compute this?
+                        return 1000f;
+                    }
+                };
+                return new ConstantScoreScorer(this, score(), scoreMode, twoPhase);
+            }
+
+            @Override
+            public boolean isCacheable(LeafReaderContext ctx) {
+                return false;
+            }
+        };
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (sameClassAs(o) == false) {
+            return false;
+        }
+        DerivedFieldQuery other = (DerivedFieldQuery) o;
+        return Objects.equals(this.query, other.query) && Objects.equals(this.indexAnalyzer, other.indexAnalyzer);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(classHash(), query, indexAnalyzer);
+    }
+
+    @Override
+    public String toString(String f) {
+        return "DerivedFieldQuery (Query: [ " + query.toString(f) + "])";
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/query/ExistsQueryBuilder.java b/server/src/main/java/org/opensearch/index/query/ExistsQueryBuilder.java
index 7fd83d5753512..3011a48fbb296 100644
--- a/server/src/main/java/org/opensearch/index/query/ExistsQueryBuilder.java
+++ b/server/src/main/java/org/opensearch/index/query/ExistsQueryBuilder.java
@@ -201,6 +201,11 @@ private static Query newObjectFieldExistsQuery(QueryShardContext context, String
         BooleanQuery.Builder booleanQuery = new BooleanQuery.Builder();
         Collection<String> fields = context.simpleMatchToIndexNames(objField + ".*");
         for (String field : fields) {
+            int dotPos = field.lastIndexOf('.');
+            if (dotPos > 0 && field.charAt(dotPos + 1) == '_') {
+                // This is a subfield (e.g. prefix) of a complex field type. Skip it.
+                continue;
+            }
             Query existsQuery = context.getMapperService().fieldType(field).existsQuery(context);
             booleanQuery.add(existsQuery, Occur.SHOULD);
         }
diff --git a/server/src/main/java/org/opensearch/index/query/QueryShardContext.java b/server/src/main/java/org/opensearch/index/query/QueryShardContext.java
index f3b392559d33e..64643ad6d2c94 100644
--- a/server/src/main/java/org/opensearch/index/query/QueryShardContext.java
+++ b/server/src/main/java/org/opensearch/index/query/QueryShardContext.java
@@ -45,6 +45,7 @@
 import org.opensearch.common.TriFunction;
 import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.lucene.search.Queries;
+import org.opensearch.common.regex.Regex;
 import org.opensearch.common.util.BigArrays;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.common.ParsingException;
@@ -77,6 +78,7 @@
 
 import java.io.IOException;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -119,6 +121,8 @@ public class QueryShardContext extends QueryRewriteContext {
     private final ValuesSourceRegistry valuesSourceRegistry;
     private BitSetProducer parentFilter;
 
+    private Map<String, MappedFieldType> derivedFieldTypeMap = new HashMap<>();
+
     public QueryShardContext(
         int shardId,
         IndexSettings indexSettings,
@@ -329,7 +333,17 @@ public Map<String, Query> copyNamedQueries() {
      * type then the fields will be returned with a type prefix.
      */
     public Set<String> simpleMatchToIndexNames(String pattern) {
-        return mapperService.simpleMatchToFullName(pattern);
+        Set<String> matchingFields = mapperService.simpleMatchToFullName(pattern);
+        if (derivedFieldTypeMap != null && !derivedFieldTypeMap.isEmpty()) {
+            Set<String> matchingDerivedFields = new HashSet<>(matchingFields);
+            for (String fieldName : derivedFieldTypeMap.keySet()) {
+                if (!matchingDerivedFields.contains(fieldName) && Regex.simpleMatch(pattern, fieldName)) {
+                    matchingDerivedFields.add(fieldName);
+                }
+            }
+            return matchingDerivedFields;
+        }
+        return matchingFields;
     }
 
     /**
@@ -395,6 +409,14 @@ public ValuesSourceRegistry getValuesSourceRegistry() {
         return valuesSourceRegistry;
     }
 
+    public void setDerivedFieldTypes(Map<String, MappedFieldType> derivedFieldTypeMap) {
+        this.derivedFieldTypeMap = derivedFieldTypeMap;
+    }
+
+    public MappedFieldType getDerivedFieldType(String fieldName) {
+        return derivedFieldTypeMap == null ? null : derivedFieldTypeMap.get(fieldName);
+    }
+
     public void setAllowUnmappedFields(boolean allowUnmappedFields) {
         this.allowUnmappedFields = allowUnmappedFields;
     }
@@ -404,7 +426,11 @@ public void setMapUnmappedFieldAsString(boolean mapUnmappedFieldAsString) {
     }
 
     MappedFieldType failIfFieldMappingNotFound(String name, MappedFieldType fieldMapping) {
-        if (fieldMapping != null || allowUnmappedFields) {
+        if (fieldMapping != null) {
+            return fieldMapping;
+        } else if (getDerivedFieldType(name) != null) {
+            return getDerivedFieldType(name);
+        } else if (allowUnmappedFields) {
             return fieldMapping;
         } else if (mapUnmappedFieldAsString) {
             TextFieldMapper.Builder builder = new TextFieldMapper.Builder(name, mapperService.getIndexAnalyzers());
diff --git a/server/src/main/java/org/opensearch/index/query/VectorGeoPointShapeQueryProcessor.java b/server/src/main/java/org/opensearch/index/query/VectorGeoPointShapeQueryProcessor.java
index eb7923a5b5212..c55d88439b11f 100644
--- a/server/src/main/java/org/opensearch/index/query/VectorGeoPointShapeQueryProcessor.java
+++ b/server/src/main/java/org/opensearch/index/query/VectorGeoPointShapeQueryProcessor.java
@@ -54,6 +54,7 @@
 import org.opensearch.geometry.Polygon;
 import org.opensearch.geometry.Rectangle;
 import org.opensearch.geometry.ShapeType;
+import org.opensearch.index.mapper.DerivedFieldType;
 import org.opensearch.index.mapper.GeoPointFieldMapper;
 import org.opensearch.index.mapper.MappedFieldType;
 
@@ -78,7 +79,9 @@ public Query geoShapeQuery(Geometry shape, String fieldName, ShapeRelation relat
 
     private void validateIsGeoPointFieldType(String fieldName, QueryShardContext context) {
         MappedFieldType fieldType = context.fieldMapper(fieldName);
-        if (fieldType instanceof GeoPointFieldMapper.GeoPointFieldType == false) {
+        if (fieldType instanceof GeoPointFieldMapper.GeoPointFieldType == false
+            && !(fieldType instanceof DerivedFieldType
+                && (((DerivedFieldType) fieldType).getTypeMappedFieldType() instanceof GeoPointFieldMapper.GeoPointFieldType))) {
             throw new QueryShardException(
                 context,
                 "Expected "
diff --git a/server/src/main/java/org/opensearch/index/recovery/RemoteStoreRestoreService.java b/server/src/main/java/org/opensearch/index/recovery/RemoteStoreRestoreService.java
index 23bb4cea17a20..fe90f24b0f544 100644
--- a/server/src/main/java/org/opensearch/index/recovery/RemoteStoreRestoreService.java
+++ b/server/src/main/java/org/opensearch/index/recovery/RemoteStoreRestoreService.java
@@ -11,6 +11,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.opensearch.action.admin.cluster.remotestore.restore.RestoreRemoteStoreRequest;
+import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.ClusterStateUpdateTask;
 import org.opensearch.cluster.block.ClusterBlocks;
@@ -48,6 +49,7 @@
 import java.util.stream.Collectors;
 
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED;
+import static org.opensearch.common.util.IndexUtils.filterIndices;
 import static org.opensearch.repositories.blobstore.BlobStoreRepository.SYSTEM_REPOSITORY_SETTING;
 
 /**
@@ -158,7 +160,12 @@ public RemoteRestoreResult restore(
                 throw new IllegalStateException("Unable to restore remote index metadata", e);
             }
         } else {
-            for (String indexName : indexNames) {
+            List<String> filteredIndices = filterIndices(
+                List.of(currentState.metadata().getConcreteAllIndices()),
+                indexNames,
+                IndicesOptions.fromOptions(true, true, true, true)
+            );
+            for (String indexName : filteredIndices) {
                 IndexMetadata indexMetadata = currentState.metadata().index(indexName);
                 if (indexMetadata == null) {
                     logger.warn("Index restore is not supported for non-existent index. Skipping: {}", indexName);
diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteIndexPath.java b/server/src/main/java/org/opensearch/index/remote/RemoteIndexPath.java
new file mode 100644
index 0000000000000..89b642b79df86
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/remote/RemoteIndexPath.java
@@ -0,0 +1,166 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.remote;
+
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.common.blobstore.BlobPath;
+import org.opensearch.core.xcontent.ToXContentFragment;
+import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.core.xcontent.XContentParser;
+import org.opensearch.index.remote.RemoteStoreEnums.DataCategory;
+import org.opensearch.index.remote.RemoteStoreEnums.DataType;
+import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm;
+import org.opensearch.index.remote.RemoteStoreEnums.PathType;
+import org.opensearch.index.remote.RemoteStorePathStrategy.PathInput;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.SEGMENTS;
+import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.TRANSLOG;
+import static org.opensearch.index.remote.RemoteStoreEnums.DataType.DATA;
+import static org.opensearch.index.remote.RemoteStoreEnums.DataType.METADATA;
+import static org.opensearch.index.remote.RemoteStorePathStrategy.isCompatible;
+
+/**
+ * Remote index path information.
+ *
+ * @opensearch.internal
+ */
+@ExperimentalApi
+public class RemoteIndexPath implements ToXContentFragment {
+
+    public static final Map<DataCategory, List<DataType>> TRANSLOG_PATH = Map.of(TRANSLOG, List.of(DATA, METADATA));
+    public static final Map<DataCategory, List<DataType>> SEGMENT_PATH = Map.of(SEGMENTS, List.of(DataType.values()));
+    public static final Map<DataCategory, List<DataType>> COMBINED_PATH;
+
+    static {
+        Map<DataCategory, List<DataType>> combinedPath = new HashMap<>();
+        combinedPath.putAll(TRANSLOG_PATH);
+        combinedPath.putAll(SEGMENT_PATH);
+        COMBINED_PATH = Collections.unmodifiableMap(combinedPath);
+    }
+    public static final String DEFAULT_VERSION = "1";
+    public static final String DIR = "remote-index-path";
+    public static final String FILE_NAME_FORMAT = "remote_path_%s";
+    static final String KEY_VERSION = "version";
+    static final String KEY_INDEX_UUID = "index_uuid";
+    static final String KEY_SHARD_COUNT = "shard_count";
+    static final String KEY_PATH_CREATION_MAP = "path_creation_map";
+    static final String KEY_PATHS = "paths";
+
+    private final String version;
+    private final String indexUUID;
+    private final int shardCount;
+    private final Iterable<String> basePath;
+    private final PathType pathType;
+    private final PathHashAlgorithm pathHashAlgorithm;
+
+    /**
+     * This keeps the map of paths that would be present in the content of the index path file. For eg - It is possible
+     * that segment and translog repository can be different. For this use case, we have either segment or translog as the
+     * key, and list of data, metadata, and lock_files (only for segment) as the value.
+     */
+    private final Map<DataCategory, List<DataType>> pathCreationMap;
+
+    public RemoteIndexPath(
+        String indexUUID,
+        int shardCount,
+        Iterable<String> basePath,
+        PathType pathType,
+        PathHashAlgorithm pathHashAlgorithm,
+        Map<DataCategory, List<DataType>> pathCreationMap
+    ) {
+        if (Objects.isNull(pathCreationMap)
+            || Objects.isNull(pathType)
+            || isCompatible(pathType, pathHashAlgorithm) == false
+            || shardCount < 1
+            || Objects.isNull(basePath)
+            || pathCreationMap.isEmpty()
+            || pathCreationMap.keySet().stream().anyMatch(k -> pathCreationMap.get(k).isEmpty())) {
+            ParameterizedMessage parameterizedMessage = new ParameterizedMessage(
+                "Invalid input in RemoteIndexPath constructor indexUUID={} shardCount={} basePath={} pathType={}"
+                    + " pathHashAlgorithm={} pathCreationMap={}",
+                indexUUID,
+                shardCount,
+                basePath,
+                pathType,
+                pathHashAlgorithm,
+                pathCreationMap
+            );
+            throw new IllegalArgumentException(parameterizedMessage.getFormattedMessage());
+        }
+        boolean validMap = pathCreationMap.keySet()
+            .stream()
+            .allMatch(k -> pathCreationMap.get(k).stream().allMatch(k::isSupportedDataType));
+        if (validMap == false) {
+            throw new IllegalArgumentException(
+                new ParameterizedMessage("pathCreationMap={} is having illegal combination of category and type", pathCreationMap)
+                    .getFormattedMessage()
+            );
+        }
+        this.version = DEFAULT_VERSION;
+        this.indexUUID = indexUUID;
+        this.shardCount = shardCount;
+        this.basePath = basePath;
+        this.pathType = pathType;
+        this.pathHashAlgorithm = pathHashAlgorithm;
+        this.pathCreationMap = pathCreationMap;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.field(KEY_VERSION, version);
+        builder.field(KEY_INDEX_UUID, indexUUID);
+        builder.field(KEY_SHARD_COUNT, shardCount);
+        builder.field(PathType.NAME, pathType.name());
+        if (Objects.nonNull(pathHashAlgorithm)) {
+            builder.field(PathHashAlgorithm.NAME, pathHashAlgorithm.name());
+        }
+
+        Map<String, List<String>> pathMap = new HashMap<>();
+        for (Map.Entry<DataCategory, List<DataType>> entry : pathCreationMap.entrySet()) {
+            pathMap.put(entry.getKey().getName(), entry.getValue().stream().map(DataType::getName).collect(Collectors.toList()));
+        }
+        builder.field(KEY_PATH_CREATION_MAP);
+        builder.map(pathMap);
+        builder.startArray(KEY_PATHS);
+        for (Map.Entry<DataCategory, List<DataType>> entry : pathCreationMap.entrySet()) {
+            DataCategory dataCategory = entry.getKey();
+            for (DataType type : entry.getValue()) {
+                for (int shardNo = 0; shardNo < shardCount; shardNo++) {
+                    PathInput pathInput = PathInput.builder()
+                        .basePath(new BlobPath().add(basePath))
+                        .indexUUID(indexUUID)
+                        .shardId(Integer.toString(shardNo))
+                        .dataCategory(dataCategory)
+                        .dataType(type)
+                        .build();
+                    builder.value(pathType.path(pathInput, pathHashAlgorithm).buildAsString());
+                }
+            }
+        }
+        builder.endArray();
+        return builder;
+    }
+
+    public static RemoteIndexPath fromXContent(XContentParser ignored) {
+        throw new UnsupportedOperationException("RemoteIndexPath.fromXContent() is not supported");
+    }
+
+    String getVersion() {
+        return version;
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteIndexPathUploader.java b/server/src/main/java/org/opensearch/index/remote/RemoteIndexPathUploader.java
new file mode 100644
index 0000000000000..d736a82d57a7c
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/remote/RemoteIndexPathUploader.java
@@ -0,0 +1,306 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.remote;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.opensearch.action.LatchedActionListener;
+import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.common.UUIDs;
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.common.blobstore.BlobContainer;
+import org.opensearch.common.blobstore.BlobPath;
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.TimeValue;
+import org.opensearch.core.action.ActionListener;
+import org.opensearch.core.index.Index;
+import org.opensearch.gateway.remote.IndexMetadataUploadListener;
+import org.opensearch.gateway.remote.RemoteClusterStateService.RemoteStateTransferException;
+import org.opensearch.index.remote.RemoteStoreEnums.PathType;
+import org.opensearch.node.Node;
+import org.opensearch.node.remotestore.RemoteStoreNodeAttribute;
+import org.opensearch.repositories.RepositoriesService;
+import org.opensearch.repositories.Repository;
+import org.opensearch.repositories.blobstore.BlobStoreRepository;
+import org.opensearch.repositories.blobstore.ConfigBlobStoreFormat;
+import org.opensearch.threadpool.ThreadPool;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+
+import static org.opensearch.gateway.remote.RemoteClusterStateService.INDEX_METADATA_UPLOAD_TIMEOUT_SETTING;
+import static org.opensearch.index.remote.RemoteIndexPath.COMBINED_PATH;
+import static org.opensearch.index.remote.RemoteIndexPath.SEGMENT_PATH;
+import static org.opensearch.index.remote.RemoteIndexPath.TRANSLOG_PATH;
+import static org.opensearch.index.remote.RemoteStoreUtils.determineRemoteStorePathStrategy;
+import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteDataAttributePresent;
+import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.isRemoteStoreClusterStateEnabled;
+
+/**
+ * Uploads the remote store path for all possible combinations of {@link org.opensearch.index.remote.RemoteStoreEnums.DataCategory}
+ * and {@link org.opensearch.index.remote.RemoteStoreEnums.DataType} for each shard of an index.
+ *
+ * @opensearch.internal
+ */
+@ExperimentalApi
+public class RemoteIndexPathUploader extends IndexMetadataUploadListener {
+
+    public static final String DELIMITER = "#";
+    public static final ConfigBlobStoreFormat<RemoteIndexPath> REMOTE_INDEX_PATH_FORMAT = new ConfigBlobStoreFormat<>(
+        RemoteIndexPath.FILE_NAME_FORMAT
+    );
+
+    private static final String TIMEOUT_EXCEPTION_MSG = "Timed out waiting while uploading remote index path file for indexes=%s";
+    private static final String UPLOAD_EXCEPTION_MSG = "Exception occurred while uploading remote index paths for indexes=%s";
+    static final String TRANSLOG_REPO_NAME_KEY = Node.NODE_ATTRIBUTES.getKey()
+        + RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY;
+    static final String SEGMENT_REPO_NAME_KEY = Node.NODE_ATTRIBUTES.getKey()
+        + RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY;
+
+    private static final Logger logger = LogManager.getLogger(RemoteIndexPathUploader.class);
+
+    private final Settings settings;
+    private final boolean isRemoteDataAttributePresent;
+    private final boolean isTranslogSegmentRepoSame;
+    private final Supplier<RepositoriesService> repositoriesService;
+    private volatile TimeValue indexMetadataUploadTimeout;
+
+    private BlobStoreRepository translogRepository;
+    private BlobStoreRepository segmentRepository;
+
+    public RemoteIndexPathUploader(
+        ThreadPool threadPool,
+        Settings settings,
+        Supplier<RepositoriesService> repositoriesService,
+        ClusterSettings clusterSettings
+    ) {
+        super(threadPool, ThreadPool.Names.GENERIC);
+        this.settings = Objects.requireNonNull(settings);
+        this.repositoriesService = Objects.requireNonNull(repositoriesService);
+        isRemoteDataAttributePresent = isRemoteDataAttributePresent(settings);
+        // If the remote data attributes are not present, then there is no effect of translog and segment being same or different or null.
+        isTranslogSegmentRepoSame = isTranslogSegmentRepoSame();
+        Objects.requireNonNull(clusterSettings);
+        indexMetadataUploadTimeout = clusterSettings.get(INDEX_METADATA_UPLOAD_TIMEOUT_SETTING);
+        clusterSettings.addSettingsUpdateConsumer(INDEX_METADATA_UPLOAD_TIMEOUT_SETTING, this::setIndexMetadataUploadTimeout);
+    }
+
+    @Override
+    protected void doOnUpload(
+        List<IndexMetadata> indexMetadataList,
+        Map<String, IndexMetadata> prevIndexMetadataByName,
+        ActionListener<Void> actionListener
+    ) {
+        if (isRemoteDataAttributePresent == false) {
+            logger.trace("Skipping beforeNewIndexUpload as there are no remote indexes");
+            actionListener.onResponse(null);
+            return;
+        }
+
+        long startTime = System.nanoTime();
+        boolean success = false;
+        List<IndexMetadata> eligibleList = indexMetadataList.stream()
+            .filter(idxMd -> requiresPathUpload(idxMd, prevIndexMetadataByName.get(idxMd.getIndex().getName())))
+            .collect(Collectors.toList());
+        String indexNames = eligibleList.stream().map(IndexMetadata::getIndex).map(Index::toString).collect(Collectors.joining(","));
+        int latchCount = eligibleList.size() * (isTranslogSegmentRepoSame ? 1 : 2);
+        CountDownLatch latch = new CountDownLatch(latchCount);
+        List<Exception> exceptionList = Collections.synchronizedList(new ArrayList<>(latchCount));
+        try {
+            for (IndexMetadata indexMetadata : eligibleList) {
+                writeIndexPathAsync(indexMetadata, latch, exceptionList);
+            }
+
+            logger.trace(new ParameterizedMessage("Remote index path upload started for {}", indexNames));
+
+            try {
+                if (latch.await(indexMetadataUploadTimeout.millis(), TimeUnit.MILLISECONDS) == false) {
+                    RemoteStateTransferException ex = new RemoteStateTransferException(
+                        String.format(Locale.ROOT, TIMEOUT_EXCEPTION_MSG, indexNames)
+                    );
+                    exceptionList.forEach(ex::addSuppressed);
+                    actionListener.onFailure(ex);
+                    return;
+                }
+            } catch (InterruptedException exception) {
+                exceptionList.forEach(exception::addSuppressed);
+                RemoteStateTransferException ex = new RemoteStateTransferException(
+                    String.format(Locale.ROOT, TIMEOUT_EXCEPTION_MSG, indexNames),
+                    exception
+                );
+                actionListener.onFailure(ex);
+                return;
+            }
+            if (exceptionList.size() > 0) {
+                RemoteStateTransferException ex = new RemoteStateTransferException(
+                    String.format(Locale.ROOT, UPLOAD_EXCEPTION_MSG, indexNames)
+                );
+                exceptionList.forEach(ex::addSuppressed);
+                actionListener.onFailure(ex);
+                return;
+            }
+            success = true;
+            actionListener.onResponse(null);
+        } catch (Exception exception) {
+            RemoteStateTransferException ex = new RemoteStateTransferException(
+                String.format(Locale.ROOT, UPLOAD_EXCEPTION_MSG, indexNames),
+                exception
+            );
+            exceptionList.forEach(ex::addSuppressed);
+            actionListener.onFailure(ex);
+        } finally {
+            long tookTimeNs = System.nanoTime() - startTime;
+            logger.trace(new ParameterizedMessage("executed beforeNewIndexUpload status={} tookTimeNs={}", success, tookTimeNs));
+        }
+
+    }
+
+    private void writeIndexPathAsync(IndexMetadata idxMD, CountDownLatch latch, List<Exception> exceptionList) {
+        if (isTranslogSegmentRepoSame) {
+            // If the repositories are same, then we need to upload a single file containing paths for both translog and segments.
+            writePathToRemoteStore(idxMD, translogRepository, latch, exceptionList, COMBINED_PATH);
+        } else {
+            // If the repositories are different, then we need to upload one file per segment and translog containing their individual
+            // paths.
+            writePathToRemoteStore(idxMD, translogRepository, latch, exceptionList, TRANSLOG_PATH);
+            writePathToRemoteStore(idxMD, segmentRepository, latch, exceptionList, SEGMENT_PATH);
+        }
+    }
+
+    private void writePathToRemoteStore(
+        IndexMetadata idxMD,
+        BlobStoreRepository repository,
+        CountDownLatch latch,
+        List<Exception> exceptionList,
+        Map<RemoteStoreEnums.DataCategory, List<RemoteStoreEnums.DataType>> pathCreationMap
+    ) {
+        Map<String, String> remoteCustomData = idxMD.getCustomData(IndexMetadata.REMOTE_STORE_CUSTOM_KEY);
+        PathType pathType = PathType.valueOf(remoteCustomData.get(PathType.NAME));
+        RemoteStoreEnums.PathHashAlgorithm hashAlgorithm = RemoteStoreEnums.PathHashAlgorithm.valueOf(
+            remoteCustomData.get(RemoteStoreEnums.PathHashAlgorithm.NAME)
+        );
+        String indexUUID = idxMD.getIndexUUID();
+        int shardCount = idxMD.getNumberOfShards();
+        BlobPath basePath = repository.basePath();
+        BlobContainer blobContainer = repository.blobStore().blobContainer(basePath.add(RemoteIndexPath.DIR));
+        ActionListener<Void> actionListener = getUploadPathLatchedActionListener(idxMD, latch, exceptionList, pathCreationMap);
+        try {
+            RemoteIndexPath remoteIndexPath = new RemoteIndexPath(
+                indexUUID,
+                shardCount,
+                basePath,
+                pathType,
+                hashAlgorithm,
+                pathCreationMap
+            );
+            String fileName = generateFileName(indexUUID, idxMD.getVersion(), remoteIndexPath.getVersion());
+            REMOTE_INDEX_PATH_FORMAT.writeAsyncWithUrgentPriority(remoteIndexPath, blobContainer, fileName, actionListener);
+        } catch (IOException ioException) {
+            RemoteStateTransferException ex = new RemoteStateTransferException(
+                String.format(Locale.ROOT, UPLOAD_EXCEPTION_MSG, List.of(idxMD.getIndex().getName())),
+                ioException
+            );
+            actionListener.onFailure(ex);
+        }
+    }
+
+    private Repository validateAndGetRepository(String repoSetting) {
+        final String repo = settings.get(repoSetting);
+        assert repo != null : "Remote " + repoSetting + " repository is not configured";
+        final Repository repository = repositoriesService.get().repository(repo);
+        assert repository instanceof BlobStoreRepository : "Repository should be instance of BlobStoreRepository";
+        return repository;
+    }
+
+    public void start() {
+        assert isRemoteStoreClusterStateEnabled(settings) == true : "Remote cluster state is not enabled";
+        if (isRemoteDataAttributePresent == false) {
+            // If remote store data attributes are not present than we skip this.
+            return;
+        }
+        translogRepository = (BlobStoreRepository) validateAndGetRepository(TRANSLOG_REPO_NAME_KEY);
+        segmentRepository = (BlobStoreRepository) validateAndGetRepository(SEGMENT_REPO_NAME_KEY);
+    }
+
+    private boolean isTranslogSegmentRepoSame() {
+        // TODO - The current comparison checks the repository name. But it is also possible that the repository are same
+        // by attributes, but different by name. We need to handle this.
+        String translogRepoName = settings.get(TRANSLOG_REPO_NAME_KEY);
+        String segmentRepoName = settings.get(SEGMENT_REPO_NAME_KEY);
+        return Objects.equals(translogRepoName, segmentRepoName);
+    }
+
+    private LatchedActionListener<Void> getUploadPathLatchedActionListener(
+        IndexMetadata indexMetadata,
+        CountDownLatch latch,
+        List<Exception> exceptionList,
+        Map<RemoteStoreEnums.DataCategory, List<RemoteStoreEnums.DataType>> pathCreationMap
+    ) {
+        return new LatchedActionListener<>(
+            ActionListener.wrap(
+                resp -> logger.trace(
+                    new ParameterizedMessage("Index path uploaded for {} indexMetadata={}", pathCreationMap, indexMetadata)
+                ),
+                ex -> {
+                    logger.error(
+                        new ParameterizedMessage(
+                            "Exception during Index path upload for {} indexMetadata={}",
+                            pathCreationMap,
+                            indexMetadata
+                        ),
+                        ex
+                    );
+                    exceptionList.add(ex);
+                }
+            ),
+            latch
+        );
+    }
+
+    /**
+     * This method checks if the index metadata has attributes that calls for uploading the index path for remote store
+     * uploads. It checks if the remote store path type is {@code HASHED_PREFIX} and returns true if so.
+     */
+    private boolean requiresPathUpload(IndexMetadata indexMetadata, IndexMetadata prevIndexMetadata) {
+        PathType pathType = determineRemoteStorePathStrategy(indexMetadata).getType();
+        PathType prevPathType = Objects.nonNull(prevIndexMetadata) ? determineRemoteStorePathStrategy(prevIndexMetadata).getType() : null;
+        // If previous metadata is null or previous path type is not hashed_prefix, and along with new path type being
+        // hashed_prefix, then this can mean any of the following -
+        // 1. This is creation of remote index with hashed_prefix
+        // 2. We are enabling cluster state for the very first time with multiple indexes having hashed_prefix path type.
+        // 3. A docrep index is being migrated to being remote store index.
+        return pathType == PathType.HASHED_PREFIX && (Objects.isNull(prevPathType) || prevPathType != PathType.HASHED_PREFIX);
+    }
+
+    private void setIndexMetadataUploadTimeout(TimeValue newIndexMetadataUploadTimeout) {
+        this.indexMetadataUploadTimeout = newIndexMetadataUploadTimeout;
+    }
+
+    /**
+     * Creates a file name by combining index uuid, index metadata version and file version. # has been chosen as the
+     * delimiter since it does not collide with any possible letters in file name. The random base64 uuid is added to
+     * ensure that the file does not get overwritten. We do check if translog and segment repo are same by name, but
+     * it is possible that a user configures same repo by different name for translog and segment in which case, this
+     * will lead to file not being overwritten.
+     */
+    private String generateFileName(String indexUUID, long indexMetadataVersion, String fileVersion) {
+        return String.join(DELIMITER, indexUUID, Long.toString(indexMetadataVersion), fileVersion, UUIDs.randomBase64UUID());
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteMigrationIndexMetadataUpdater.java b/server/src/main/java/org/opensearch/index/remote/RemoteMigrationIndexMetadataUpdater.java
new file mode 100644
index 0000000000000..761fa20ea64e5
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/remote/RemoteMigrationIndexMetadataUpdater.java
@@ -0,0 +1,181 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.remote;
+
+import org.apache.logging.log4j.Logger;
+import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.cluster.node.DiscoveryNodes;
+import org.opensearch.cluster.routing.IndexRoutingTable;
+import org.opensearch.cluster.routing.RoutingTable;
+import org.opensearch.cluster.routing.ShardRouting;
+import org.opensearch.cluster.routing.ShardRoutingState;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.index.remote.RemoteStoreEnums.PathType;
+import org.opensearch.indices.replication.common.ReplicationType;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.opensearch.cluster.metadata.IndexMetadata.REMOTE_STORE_CUSTOM_KEY;
+import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY;
+import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED;
+import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY;
+import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE;
+import static org.opensearch.index.remote.RemoteStoreUtils.determineRemoteStorePathStrategyDuringMigration;
+import static org.opensearch.index.remote.RemoteStoreUtils.getRemoteStoreRepoName;
+import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY;
+import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY;
+
+/**
+ * Utils for checking and mutating cluster state during remote migration
+ *
+ * @opensearch.internal
+ */
+public class RemoteMigrationIndexMetadataUpdater {
+    private final DiscoveryNodes discoveryNodes;
+    private final RoutingTable routingTable;
+    private final Settings clusterSettings;
+    private final IndexMetadata indexMetadata;
+    private final Logger logger;
+
+    public RemoteMigrationIndexMetadataUpdater(
+        DiscoveryNodes discoveryNodes,
+        RoutingTable routingTable,
+        IndexMetadata indexMetadata,
+        Settings clusterSettings,
+        Logger logger
+
+    ) {
+        this.discoveryNodes = discoveryNodes;
+        this.routingTable = routingTable;
+        this.clusterSettings = clusterSettings;
+        this.indexMetadata = indexMetadata;
+        this.logger = logger;
+    }
+
+    /**
+     * During docrep to remote store migration, applies the following remote store based index settings
+     * once all shards of an index have moved over to remote store enabled nodes
+     * <br>
+     * Also appends the requisite Remote Store Path based custom metadata to the existing index metadata
+     */
+    public void maybeAddRemoteIndexSettings(IndexMetadata.Builder indexMetadataBuilder, String index) {
+        Settings currentIndexSettings = indexMetadata.getSettings();
+        if (needsRemoteIndexSettingsUpdate(routingTable.indicesRouting().get(index), discoveryNodes, currentIndexSettings)) {
+            logger.info(
+                "Index {} does not have remote store based index settings but all primary shards and STARTED replica shards have moved to remote enabled nodes. Applying remote store settings to the index",
+                index
+            );
+            Map<String, String> remoteRepoNames = getRemoteStoreRepoName(discoveryNodes);
+            String segmentRepoName = remoteRepoNames.get(REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY);
+            String tlogRepoName = remoteRepoNames.get(REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY);
+            assert Objects.nonNull(segmentRepoName) && Objects.nonNull(tlogRepoName) : "Remote repo names cannot be null";
+            Settings.Builder indexSettingsBuilder = Settings.builder().put(currentIndexSettings);
+            updateRemoteStoreSettings(indexSettingsBuilder, segmentRepoName, tlogRepoName);
+            indexMetadataBuilder.settings(indexSettingsBuilder);
+            indexMetadataBuilder.settingsVersion(1 + indexMetadata.getVersion());
+        } else {
+            logger.debug("Index {} does not satisfy criteria for applying remote store settings", index);
+        }
+    }
+
+    /**
+     * Returns <code>true</code> iff all the below conditions are true:
+     * - All primary shards are in {@link ShardRoutingState#STARTED} state and are in remote store enabled nodes
+     * - No replica shard in {@link ShardRoutingState#RELOCATING} state
+     * - All {@link ShardRoutingState#STARTED} replica shards are in remote store enabled nodes
+     *
+     * @param indexRoutingTable    current {@link IndexRoutingTable} from cluster state
+     * @param discoveryNodes       set of discovery nodes from cluster state
+     * @param currentIndexSettings current {@link IndexMetadata} from cluster state
+     * @return <code>true</code> or <code>false</code> depending on the met conditions
+     */
+    private boolean needsRemoteIndexSettingsUpdate(
+        IndexRoutingTable indexRoutingTable,
+        DiscoveryNodes discoveryNodes,
+        Settings currentIndexSettings
+    ) {
+        assert currentIndexSettings != null : "IndexMetadata for a shard cannot be null";
+        if (indexHasRemoteStoreSettings(currentIndexSettings) == false) {
+            boolean allPrimariesStartedAndOnRemote = indexRoutingTable.shardsMatchingPredicate(ShardRouting::primary)
+                .stream()
+                .allMatch(shardRouting -> shardRouting.started() && discoveryNodes.get(shardRouting.currentNodeId()).isRemoteStoreNode());
+            List<ShardRouting> replicaShards = indexRoutingTable.shardsMatchingPredicate(shardRouting -> shardRouting.primary() == false);
+            boolean noRelocatingReplicas = replicaShards.stream().noneMatch(ShardRouting::relocating);
+            boolean allStartedReplicasOnRemote = replicaShards.stream()
+                .filter(ShardRouting::started)
+                .allMatch(shardRouting -> discoveryNodes.get(shardRouting.currentNodeId()).isRemoteStoreNode());
+            return allPrimariesStartedAndOnRemote && noRelocatingReplicas && allStartedReplicasOnRemote;
+        }
+        return false;
+    }
+
+    /**
+     * Updates the remote store path strategy metadata for the index when it is migrating to remote.
+     * This is run during state change of each shard copy when the cluster is in `MIXED` mode and the direction of migration is `REMOTE_STORE`
+     * Should not interfere with docrep functionality even if the index is in docrep nodes since this metadata
+     * is not used anywhere in the docrep flow
+     * Checks are in place to make this execution no-op if the index metadata is already present.
+     *
+     * @param indexMetadataBuilder Mutated {@link IndexMetadata.Builder} having the previous state updates
+     * @param index                index name
+     */
+    public void maybeUpdateRemoteStorePathStrategy(IndexMetadata.Builder indexMetadataBuilder, String index) {
+        if (indexHasRemotePathMetadata(indexMetadata) == false) {
+            logger.info("Adding remote store path strategy for index [{}] during migration", index);
+            indexMetadataBuilder.putCustom(
+                REMOTE_STORE_CUSTOM_KEY,
+                determineRemoteStorePathStrategyDuringMigration(clusterSettings, discoveryNodes)
+            );
+        } else {
+            logger.debug("Index {} already has remote store path strategy", index);
+        }
+    }
+
+    public static boolean indexHasAllRemoteStoreRelatedMetadata(IndexMetadata indexMetadata) {
+        return indexHasRemoteStoreSettings(indexMetadata.getSettings()) && indexHasRemotePathMetadata(indexMetadata);
+    }
+
+    /**
+     * Assert current index settings have:
+     * - index.remote_store.enabled == true
+     * - index.remote_store.segment.repository != null
+     * - index.remote_store.translog.repository != null
+     * - index.replication.type == SEGMENT
+     *
+     * @param indexSettings Current index settings
+     * @return <code>true</code> if all above conditions match. <code>false</code> otherwise
+     */
+    public static boolean indexHasRemoteStoreSettings(Settings indexSettings) {
+        return IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING.exists(indexSettings)
+            && IndexMetadata.INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING.exists(indexSettings)
+            && IndexMetadata.INDEX_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING.exists(indexSettings)
+            && IndexMetadata.INDEX_REPLICATION_TYPE_SETTING.get(indexSettings) == ReplicationType.SEGMENT;
+    }
+
+    /**
+     * Asserts current index metadata customs has the {@link IndexMetadata#REMOTE_STORE_CUSTOM_KEY} key.
+     * If it does, checks if the path_type sub-key is present
+     *
+     * @param indexMetadata Current index metadata
+     * @return <code>true</code> if all above conditions match. <code>false</code> otherwise
+     */
+    public static boolean indexHasRemotePathMetadata(IndexMetadata indexMetadata) {
+        Map<String, String> customMetadata = indexMetadata.getCustomData(REMOTE_STORE_CUSTOM_KEY);
+        return Objects.nonNull(customMetadata) && Objects.nonNull(customMetadata.get(PathType.NAME));
+    }
+
+    public static void updateRemoteStoreSettings(Settings.Builder settingsBuilder, String segmentRepository, String translogRepository) {
+        settingsBuilder.put(SETTING_REMOTE_STORE_ENABLED, true)
+            .put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT)
+            .put(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY, segmentRepository)
+            .put(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY, translogRepository);
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteStoreEnums.java b/server/src/main/java/org/opensearch/index/remote/RemoteStoreEnums.java
new file mode 100644
index 0000000000000..c1ac74724e405
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/remote/RemoteStoreEnums.java
@@ -0,0 +1,301 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.remote;
+
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.common.annotation.PublicApi;
+import org.opensearch.common.blobstore.BlobPath;
+import org.opensearch.common.hash.FNV1a;
+import org.opensearch.index.remote.RemoteStorePathStrategy.PathInput;
+
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
+import static java.util.Collections.unmodifiableMap;
+import static org.opensearch.index.remote.RemoteStoreEnums.DataType.DATA;
+import static org.opensearch.index.remote.RemoteStoreEnums.DataType.METADATA;
+import static org.opensearch.index.remote.RemoteStoreUtils.longToCompositeBase64AndBinaryEncoding;
+import static org.opensearch.index.remote.RemoteStoreUtils.longToUrlBase64;
+
+/**
+ * This class contains the different enums related to remote store like data categories and types, path types
+ * and hashing algorithm.
+ *
+ * @opensearch.api
+ */
+@ExperimentalApi
+public class RemoteStoreEnums {
+
+    /**
+     * Categories of the data in Remote store.
+     */
+    @PublicApi(since = "2.14.0")
+    @ExperimentalApi
+    public enum DataCategory {
+        SEGMENTS("segments", Set.of(DataType.values())),
+        TRANSLOG("translog", Set.of(DATA, METADATA));
+
+        private final String name;
+        private final Set<DataType> supportedDataTypes;
+
+        DataCategory(String name, Set<DataType> supportedDataTypes) {
+            this.name = name;
+            this.supportedDataTypes = supportedDataTypes;
+        }
+
+        public boolean isSupportedDataType(DataType dataType) {
+            return supportedDataTypes.contains(dataType);
+        }
+
+        public String getName() {
+            return name;
+        }
+    }
+
+    /**
+     * Types of data in remote store.
+     */
+    @PublicApi(since = "2.14.0")
+    @ExperimentalApi
+    public enum DataType {
+        DATA("data"),
+        METADATA("metadata"),
+        LOCK_FILES("lock_files");
+
+        private final String name;
+
+        DataType(String name) {
+            this.name = name;
+        }
+
+        public String getName() {
+            return name;
+        }
+    }
+
+    /**
+     * Enumerates the types of remote store paths resolution techniques supported by OpenSearch.
+     * For more information, see <a href="https://github.com/opensearch-project/OpenSearch/issues/12567">Github issue #12567</a>.
+     */
+    @PublicApi(since = "2.14.0")
+    @ExperimentalApi
+    public enum PathType {
+        FIXED(0) {
+            @Override
+            public BlobPath generatePath(PathInput pathInput, PathHashAlgorithm hashAlgorithm) {
+                assert Objects.isNull(hashAlgorithm) : "hashAlgorithm is expected to be null with fixed remote store path type";
+                // Hash algorithm is not used in FIXED path type
+                return pathInput.basePath()
+                    .add(pathInput.indexUUID())
+                    .add(pathInput.shardId())
+                    .add(pathInput.dataCategory().getName())
+                    .add(pathInput.dataType().getName());
+            }
+
+            @Override
+            boolean requiresHashAlgorithm() {
+                return false;
+            }
+        },
+        HASHED_PREFIX(1) {
+            @Override
+            public BlobPath generatePath(PathInput pathInput, PathHashAlgorithm hashAlgorithm) {
+                assert Objects.nonNull(hashAlgorithm) : "hashAlgorithm is expected to be non-null";
+                return BlobPath.cleanPath()
+                    .add(hashAlgorithm.hash(pathInput))
+                    .add(pathInput.basePath())
+                    .add(pathInput.indexUUID())
+                    .add(pathInput.shardId())
+                    .add(pathInput.dataCategory().getName())
+                    .add(pathInput.dataType().getName());
+            }
+
+            @Override
+            boolean requiresHashAlgorithm() {
+                return true;
+            }
+        },
+        HASHED_INFIX(2) {
+            @Override
+            public BlobPath generatePath(PathInput pathInput, PathHashAlgorithm hashAlgorithm) {
+                assert Objects.nonNull(hashAlgorithm) : "hashAlgorithm is expected to be non-null";
+                return pathInput.basePath()
+                    .add(hashAlgorithm.hash(pathInput))
+                    .add(pathInput.indexUUID())
+                    .add(pathInput.shardId())
+                    .add(pathInput.dataCategory().getName())
+                    .add(pathInput.dataType().getName());
+            }
+
+            @Override
+            boolean requiresHashAlgorithm() {
+                return true;
+            }
+        };
+
+        private final int code;
+
+        PathType(int code) {
+            this.code = code;
+        }
+
+        public int getCode() {
+            return code;
+        }
+
+        private static final Map<Integer, PathType> CODE_TO_ENUM;
+
+        static {
+            PathType[] values = values();
+            Map<Integer, PathType> codeToStatus = new HashMap<>(values.length);
+            for (PathType value : values) {
+                int code = value.code;
+                if (codeToStatus.containsKey(code)) {
+                    throw new IllegalStateException(
+                        new ParameterizedMessage("{} has same code as {}", codeToStatus.get(code), value).getFormattedMessage()
+                    );
+                }
+                codeToStatus.put(code, value);
+            }
+            CODE_TO_ENUM = unmodifiableMap(codeToStatus);
+        }
+
+        /**
+         * Turn a status code into a {@link PathType}.
+         */
+        public static PathType fromCode(int code) {
+            return CODE_TO_ENUM.get(code);
+        }
+
+        /**
+         * This method generates the path for the given path input which constitutes multiple fields and characteristics
+         * of the data.
+         *
+         * @param pathInput     input.
+         * @param hashAlgorithm hashing algorithm.
+         * @return the blob path for the path input.
+         */
+        public BlobPath path(PathInput pathInput, PathHashAlgorithm hashAlgorithm) {
+            DataCategory dataCategory = pathInput.dataCategory();
+            DataType dataType = pathInput.dataType();
+            assert dataCategory.isSupportedDataType(dataType) : "category:"
+                + dataCategory
+                + " type:"
+                + dataType
+                + " are not supported together";
+            return generatePath(pathInput, hashAlgorithm);
+        }
+
+        protected abstract BlobPath generatePath(PathInput pathInput, PathHashAlgorithm hashAlgorithm);
+
+        abstract boolean requiresHashAlgorithm();
+
+        public static PathType parseString(String pathType) {
+            try {
+                return PathType.valueOf(pathType.toUpperCase(Locale.ROOT));
+            } catch (IllegalArgumentException | NullPointerException e) {
+                // IllegalArgumentException is thrown when the input does not match any enum name
+                // NullPointerException is thrown when the input is null
+                throw new IllegalArgumentException("Could not parse PathType for [" + pathType + "]");
+            }
+        }
+
+        /**
+         * This string is used as key for storing information in the custom data in index settings.
+         */
+        public static final String NAME = "path_type";
+
+    }
+
+    /**
+     * Type of hashes supported for path types that have hashing.
+     */
+    @PublicApi(since = "2.14.0")
+    @ExperimentalApi
+    public enum PathHashAlgorithm {
+
+        FNV_1A_BASE64(0) {
+            @Override
+            String hash(PathInput pathInput) {
+                String input = pathInput.indexUUID() + pathInput.shardId() + pathInput.dataCategory().getName() + pathInput.dataType()
+                    .getName();
+                long hash = FNV1a.hash64(input);
+                return longToUrlBase64(hash);
+            }
+        },
+        /**
+         * This hash algorithm will generate a hash value which will use 1st 6 bits to create bas64 character and next 14
+         * bits to create binary string.
+         */
+        FNV_1A_COMPOSITE_1(1) {
+            @Override
+            String hash(PathInput pathInput) {
+                String input = pathInput.indexUUID() + pathInput.shardId() + pathInput.dataCategory().getName() + pathInput.dataType()
+                    .getName();
+                long hash = FNV1a.hash64(input);
+                return longToCompositeBase64AndBinaryEncoding(hash, 20);
+            }
+        };
+
+        private final int code;
+
+        PathHashAlgorithm(int code) {
+            this.code = code;
+        }
+
+        public int getCode() {
+            return code;
+        }
+
+        private static final Map<Integer, PathHashAlgorithm> CODE_TO_ENUM;
+
+        static {
+            PathHashAlgorithm[] values = values();
+            Map<Integer, PathHashAlgorithm> codeToStatus = new HashMap<>(values.length);
+            for (PathHashAlgorithm value : values) {
+                int code = value.code;
+                if (codeToStatus.containsKey(code)) {
+                    throw new IllegalStateException(
+                        new ParameterizedMessage("{} has same code as {}", codeToStatus.get(code), value).getFormattedMessage()
+                    );
+                }
+                codeToStatus.put(code, value);
+            }
+            CODE_TO_ENUM = unmodifiableMap(codeToStatus);
+        }
+
+        /**
+         * Turn a status code into a {@link PathHashAlgorithm}.
+         */
+        public static PathHashAlgorithm fromCode(int code) {
+            return CODE_TO_ENUM.get(code);
+        }
+
+        abstract String hash(PathInput pathInput);
+
+        public static PathHashAlgorithm parseString(String pathHashAlgorithm) {
+            try {
+                return PathHashAlgorithm.valueOf(pathHashAlgorithm.toUpperCase(Locale.ROOT));
+            } catch (IllegalArgumentException | NullPointerException e) {
+                // IllegalArgumentException is thrown when the input does not match any enum name
+                // NullPointerException is thrown when the input is null
+                throw new IllegalArgumentException("Could not parse PathHashAlgorithm for [" + pathHashAlgorithm + "]");
+            }
+        }
+
+        /**
+         * This string is used as key for storing information in the custom data in index settings.
+         */
+        public static final String NAME = "path_hash_algorithm";
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteStorePathResolver.java b/server/src/main/java/org/opensearch/index/remote/RemoteStorePathResolver.java
deleted file mode 100644
index 6e8126fcce0ca..0000000000000
--- a/server/src/main/java/org/opensearch/index/remote/RemoteStorePathResolver.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.index.remote;
-
-import org.opensearch.common.settings.ClusterSettings;
-import org.opensearch.indices.IndicesService;
-
-/**
- * Determines the {@link RemoteStorePathType} at the time of index metadata creation.
- *
- * @opensearch.internal
- */
-public class RemoteStorePathResolver {
-
-    private final ClusterSettings clusterSettings;
-
-    public RemoteStorePathResolver(ClusterSettings clusterSettings) {
-        this.clusterSettings = clusterSettings;
-    }
-
-    public RemoteStorePathType resolveType() {
-        return clusterSettings.get(IndicesService.CLUSTER_REMOTE_STORE_PATH_PREFIX_TYPE_SETTING);
-    }
-}
diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteStorePathStrategy.java b/server/src/main/java/org/opensearch/index/remote/RemoteStorePathStrategy.java
new file mode 100644
index 0000000000000..c58f6c3faac84
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/remote/RemoteStorePathStrategy.java
@@ -0,0 +1,167 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.remote;
+
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.common.annotation.PublicApi;
+import org.opensearch.common.blobstore.BlobPath;
+import org.opensearch.index.remote.RemoteStoreEnums.DataCategory;
+import org.opensearch.index.remote.RemoteStoreEnums.DataType;
+import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm;
+import org.opensearch.index.remote.RemoteStoreEnums.PathType;
+
+import java.util.Objects;
+
+/**
+ * This class wraps internal details on the remote store path for an index.
+ *
+ * @opensearch.internal
+ */
+@PublicApi(since = "2.14.0")
+@ExperimentalApi
+public class RemoteStorePathStrategy {
+
+    private final PathType type;
+
+    @Nullable
+    private final PathHashAlgorithm hashAlgorithm;
+
+    public RemoteStorePathStrategy(PathType type) {
+        this(type, null);
+    }
+
+    public RemoteStorePathStrategy(PathType type, PathHashAlgorithm hashAlgorithm) {
+        Objects.requireNonNull(type, "pathType can not be null");
+        if (isCompatible(type, hashAlgorithm) == false) {
+            throw new IllegalArgumentException(
+                new ParameterizedMessage("pathType={} pathHashAlgorithm={} are incompatible", type, hashAlgorithm).getFormattedMessage()
+            );
+        }
+        this.type = type;
+        this.hashAlgorithm = hashAlgorithm;
+    }
+
+    public static boolean isCompatible(PathType type, PathHashAlgorithm hashAlgorithm) {
+        return (type.requiresHashAlgorithm() == false && Objects.isNull(hashAlgorithm))
+            || (type.requiresHashAlgorithm() && Objects.nonNull(hashAlgorithm));
+    }
+
+    public PathType getType() {
+        return type;
+    }
+
+    public PathHashAlgorithm getHashAlgorithm() {
+        return hashAlgorithm;
+    }
+
+    @Override
+    public String toString() {
+        return "RemoteStorePathStrategy{" + "type=" + type + ", hashAlgorithm=" + hashAlgorithm + '}';
+    }
+
+    public BlobPath generatePath(PathInput pathInput) {
+        return type.path(pathInput, hashAlgorithm);
+    }
+
+    /**
+     * Wrapper class for the input required to generate path for remote store uploads.
+     * @opensearch.internal
+     */
+    @PublicApi(since = "2.14.0")
+    @ExperimentalApi
+    public static class PathInput {
+        private final BlobPath basePath;
+        private final String indexUUID;
+        private final String shardId;
+        private final DataCategory dataCategory;
+        private final DataType dataType;
+
+        public PathInput(BlobPath basePath, String indexUUID, String shardId, DataCategory dataCategory, DataType dataType) {
+            this.basePath = Objects.requireNonNull(basePath);
+            this.indexUUID = Objects.requireNonNull(indexUUID);
+            this.shardId = Objects.requireNonNull(shardId);
+            this.dataCategory = Objects.requireNonNull(dataCategory);
+            this.dataType = Objects.requireNonNull(dataType);
+        }
+
+        BlobPath basePath() {
+            return basePath;
+        }
+
+        String indexUUID() {
+            return indexUUID;
+        }
+
+        String shardId() {
+            return shardId;
+        }
+
+        DataCategory dataCategory() {
+            return dataCategory;
+        }
+
+        DataType dataType() {
+            return dataType;
+        }
+
+        /**
+         * Returns a new builder for {@link PathInput}.
+         */
+        public static Builder builder() {
+            return new Builder();
+        }
+
+        /**
+         * Builder for {@link PathInput}.
+         *
+         * @opensearch.internal
+         */
+        @PublicApi(since = "2.14.0")
+        @ExperimentalApi
+        public static class Builder {
+            private BlobPath basePath;
+            private String indexUUID;
+            private String shardId;
+            private DataCategory dataCategory;
+            private DataType dataType;
+
+            public Builder basePath(BlobPath basePath) {
+                this.basePath = basePath;
+                return this;
+            }
+
+            public Builder indexUUID(String indexUUID) {
+                this.indexUUID = indexUUID;
+                return this;
+            }
+
+            public Builder shardId(String shardId) {
+                this.shardId = shardId;
+                return this;
+            }
+
+            public Builder dataCategory(DataCategory dataCategory) {
+                this.dataCategory = dataCategory;
+                return this;
+            }
+
+            public Builder dataType(DataType dataType) {
+                this.dataType = dataType;
+                return this;
+            }
+
+            public PathInput build() {
+                return new PathInput(basePath, indexUUID, shardId, dataCategory, dataType);
+            }
+        }
+    }
+
+}
diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteStorePathStrategyResolver.java b/server/src/main/java/org/opensearch/index/remote/RemoteStorePathStrategyResolver.java
new file mode 100644
index 0000000000000..178de406ed681
--- /dev/null
+++ b/server/src/main/java/org/opensearch/index/remote/RemoteStorePathStrategyResolver.java
@@ -0,0 +1,44 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.remote;
+
+import org.opensearch.Version;
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm;
+import org.opensearch.index.remote.RemoteStoreEnums.PathType;
+import org.opensearch.indices.RemoteStoreSettings;
+
+import java.util.function.Supplier;
+
+/**
+ * Determines the {@link RemoteStorePathStrategy} at the time of index metadata creation.
+ *
+ * @opensearch.internal
+ */
+@ExperimentalApi
+public class RemoteStorePathStrategyResolver {
+
+    private final RemoteStoreSettings remoteStoreSettings;
+    private final Supplier<Version> minNodeVersionSupplier;
+
+    public RemoteStorePathStrategyResolver(RemoteStoreSettings remoteStoreSettings, Supplier<Version> minNodeVersionSupplier) {
+        this.remoteStoreSettings = remoteStoreSettings;
+        this.minNodeVersionSupplier = minNodeVersionSupplier;
+    }
+
+    public RemoteStorePathStrategy get() {
+        PathType pathType;
+        PathHashAlgorithm pathHashAlgorithm;
+        // Min node version check ensures that we are enabling the new prefix type only when all the nodes understand it.
+        pathType = Version.V_2_14_0.compareTo(minNodeVersionSupplier.get()) <= 0 ? remoteStoreSettings.getPathType() : PathType.FIXED;
+        // If the path type is fixed, hash algorithm is not applicable.
+        pathHashAlgorithm = pathType == PathType.FIXED ? null : remoteStoreSettings.getPathHashAlgorithm();
+        return new RemoteStorePathStrategy(pathType, pathHashAlgorithm);
+    }
+}
diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteStorePathType.java b/server/src/main/java/org/opensearch/index/remote/RemoteStorePathType.java
deleted file mode 100644
index a64e07ab1f66f..0000000000000
--- a/server/src/main/java/org/opensearch/index/remote/RemoteStorePathType.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.index.remote;
-
-import java.util.Locale;
-
-/**
- * Enumerates the types of remote store paths resolution techniques supported by OpenSearch.
- * For more information, see <a href="https://github.com/opensearch-project/OpenSearch/issues/12567">Github issue #12567</a>.
- *
- * @opensearch.internal
- */
-public enum RemoteStorePathType {
-
-    FIXED,
-    HASHED_PREFIX;
-
-    public static RemoteStorePathType parseString(String remoteStoreBlobPathType) {
-        try {
-            return RemoteStorePathType.valueOf(remoteStoreBlobPathType.toUpperCase(Locale.ROOT));
-        } catch (IllegalArgumentException e) {
-            throw new IllegalArgumentException("Could not parse RemoteStorePathType for [" + remoteStoreBlobPathType + "]");
-        }
-    }
-
-    /**
-     * This string is used as key for storing information in the custom data in index settings.
-     */
-    public static final String NAME = "path_type";
-}
diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteStorePressureService.java b/server/src/main/java/org/opensearch/index/remote/RemoteStorePressureService.java
index 33cd40f802d43..52f83096dc08d 100644
--- a/server/src/main/java/org/opensearch/index/remote/RemoteStorePressureService.java
+++ b/server/src/main/java/org/opensearch/index/remote/RemoteStorePressureService.java
@@ -78,7 +78,9 @@ public void validateSegmentsUploadLag(ShardId shardId) {
         for (LagValidator lagValidator : lagValidators) {
             if (lagValidator.validate(remoteSegmentTransferTracker, shardId) == false) {
                 remoteSegmentTransferTracker.incrementRejectionCount(lagValidator.name());
-                throw new OpenSearchRejectedExecutionException(lagValidator.rejectionMessage(remoteSegmentTransferTracker, shardId));
+                String rejectionMessage = lagValidator.rejectionMessage(remoteSegmentTransferTracker, shardId);
+                logger.warn("Rejecting write requests for shard due to remote backpressure:  {}", rejectionMessage);
+                throw new OpenSearchRejectedExecutionException(rejectionMessage);
             }
         }
     }
diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteStoreStatsTrackerFactory.java b/server/src/main/java/org/opensearch/index/remote/RemoteStoreStatsTrackerFactory.java
index e4c7eb56d02c6..21753e68db498 100644
--- a/server/src/main/java/org/opensearch/index/remote/RemoteStoreStatsTrackerFactory.java
+++ b/server/src/main/java/org/opensearch/index/remote/RemoteStoreStatsTrackerFactory.java
@@ -68,7 +68,7 @@ public RemoteStoreStatsTrackerFactory(ClusterService clusterService, Settings se
 
     @Override
     public void afterIndexShardCreated(IndexShard indexShard) {
-        if (indexShard.indexSettings().isRemoteStoreEnabled() == false && indexShard.indexSettings().isRemoteNode() == false) {
+        if (indexShard.indexSettings().isRemoteStoreEnabled() == false && indexShard.indexSettings().isAssignedOnRemoteNode() == false) {
             return;
         }
         ShardId shardId = indexShard.shardId();
diff --git a/server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java b/server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java
index b4c33d781af86..27b1b88034573 100644
--- a/server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java
+++ b/server/src/main/java/org/opensearch/index/remote/RemoteStoreUtils.java
@@ -8,26 +8,50 @@
 
 package org.opensearch.index.remote;
 
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.opensearch.Version;
+import org.opensearch.cluster.ClusterState;
+import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.cluster.node.DiscoveryNodes;
 import org.opensearch.common.collect.Tuple;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.node.remotestore.RemoteStoreNodeAttribute;
 
+import java.nio.ByteBuffer;
 import java.util.Arrays;
+import java.util.Base64;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
 import java.util.function.Function;
 
+import static org.opensearch.indices.RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING;
+import static org.opensearch.indices.RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING;
+
 /**
  * Utils for remote store
  *
  * @opensearch.internal
  */
 public class RemoteStoreUtils {
+    private static final Logger logger = LogManager.getLogger(RemoteStoreUtils.class);
     public static final int LONG_MAX_LENGTH = String.valueOf(Long.MAX_VALUE).length();
 
+    /**
+     * URL safe base 64 character set. This must not be changed as this is used in deriving the base64 equivalent of binary.
+     */
+    static final char[] URL_BASE64_CHARSET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_".toCharArray();
+
     /**
      * This method subtracts given numbers from Long.MAX_VALUE and returns a string representation of the result.
      * The resultant string is guaranteed to be of the same length that of Long.MAX_VALUE. If shorter, we add left padding
      * of 0s to the string.
+     *
      * @param num number to get the inverted long string for
      * @return String value of Long.MAX_VALUE - num
      */
@@ -44,6 +68,7 @@ public static String invertLong(long num) {
 
     /**
      * This method converts the given string into long and subtracts it from Long.MAX_VALUE
+     *
      * @param str long in string format to be inverted
      * @return long value of the invert result
      */
@@ -57,6 +82,7 @@ public static long invertLong(String str) {
 
     /**
      * Extracts the segment name from the provided segment file name
+     *
      * @param filename Segment file name to parse
      * @return Name of the segment that the segment file belongs to
      */
@@ -77,10 +103,9 @@ public static String getSegmentName(String filename) {
     }
 
     /**
-     *
      * @param mdFiles List of segment/translog metadata files
-     * @param fn Function to extract PrimaryTerm_Generation and Node Id from metadata file name .
-     *          fn returns null if node id is not part of the file name
+     * @param fn      Function to extract PrimaryTerm_Generation and Node Id from metadata file name .
+     *                fn returns null if node id is not part of the file name
      */
     public static void verifyNoMultipleWriters(List<String> mdFiles, Function<String, Tuple<String, String>> fn) {
         Map<String, String> nodesByPrimaryTermAndGen = new HashMap<>();
@@ -101,4 +126,102 @@ public static void verifyNoMultipleWriters(List<String> mdFiles, Function<String
         });
     }
 
+    /**
+     * Converts an input hash which occupies 64 bits of space into Base64 (6 bits per character) String. This must not
+     * be changed as it is used for creating path for storing remote store data on the remote store.
+     * This converts the byte array to base 64 string. `/` is replaced with `_`, `+` is replaced with `-` and `=`
+     * which is padded at the last is also removed. These characters are either used as delimiter or special character
+     * requiring special handling in some vendors. The characters present in this base64 version are [A-Za-z0-9_-].
+     * This must not be changed as it is used for creating path for storing remote store data on the remote store.
+     */
+    static String longToUrlBase64(long value) {
+        byte[] hashBytes = ByteBuffer.allocate(Long.BYTES).putLong(value).array();
+        String base64Str = Base64.getUrlEncoder().encodeToString(hashBytes);
+        return base64Str.substring(0, base64Str.length() - 1);
+    }
+
+    static long urlBase64ToLong(String base64Str) {
+        byte[] hashBytes = Base64.getUrlDecoder().decode(base64Str);
+        return ByteBuffer.wrap(hashBytes).getLong();
+    }
+
+    /**
+     * Converts an input hash which occupies 64 bits of memory into a composite encoded string. The string will have 2 parts -
+     * 1. Base 64 string and 2. Binary String. We will use the first 6 bits for creating the base 64 string.
+     * For the second part, the rest of the bits (of length {@code len}-6) will be used as is in string form.
+     */
+    static String longToCompositeBase64AndBinaryEncoding(long value, int len) {
+        if (len < 7 || len > 64) {
+            throw new IllegalArgumentException("In longToCompositeBase64AndBinaryEncoding, len must be between 7 and 64 (both inclusive)");
+        }
+        String binaryEncoding = String.format(Locale.ROOT, "%64s", Long.toBinaryString(value)).replace(' ', '0');
+        String base64Part = binaryEncoding.substring(0, 6);
+        String binaryPart = binaryEncoding.substring(6, len);
+        int base64DecimalValue = Integer.valueOf(base64Part, 2);
+        assert base64DecimalValue >= 0 && base64DecimalValue < 64;
+        return URL_BASE64_CHARSET[base64DecimalValue] + binaryPart;
+    }
+
+    /**
+     * Determines the remote store path strategy by reading the custom data map in IndexMetadata class.
+     */
+    public static RemoteStorePathStrategy determineRemoteStorePathStrategy(IndexMetadata indexMetadata) {
+        Map<String, String> remoteCustomData = indexMetadata.getCustomData(IndexMetadata.REMOTE_STORE_CUSTOM_KEY);
+        assert remoteCustomData == null || remoteCustomData.containsKey(RemoteStoreEnums.PathType.NAME);
+        if (remoteCustomData != null && remoteCustomData.containsKey(RemoteStoreEnums.PathType.NAME)) {
+            RemoteStoreEnums.PathType pathType = RemoteStoreEnums.PathType.parseString(
+                remoteCustomData.get(RemoteStoreEnums.PathType.NAME)
+            );
+            String hashAlgoStr = remoteCustomData.get(RemoteStoreEnums.PathHashAlgorithm.NAME);
+            RemoteStoreEnums.PathHashAlgorithm hashAlgorithm = Objects.nonNull(hashAlgoStr)
+                ? RemoteStoreEnums.PathHashAlgorithm.parseString(hashAlgoStr)
+                : null;
+            return new RemoteStorePathStrategy(pathType, hashAlgorithm);
+        }
+        return new RemoteStorePathStrategy(RemoteStoreEnums.PathType.FIXED);
+    }
+
+    /**
+     * Generates the remote store path type information to be added to custom data of index metadata during migration
+     *
+     * @param clusterSettings Current Cluster settings from {@link ClusterState}
+     * @param discoveryNodes Current {@link DiscoveryNodes} from the cluster state
+     * @return {@link Map} to be added as custom data in index metadata
+     */
+    public static Map<String, String> determineRemoteStorePathStrategyDuringMigration(
+        Settings clusterSettings,
+        DiscoveryNodes discoveryNodes
+    ) {
+        Version minNodeVersion = discoveryNodes.getMinNodeVersion();
+        RemoteStoreEnums.PathType pathType = Version.CURRENT.compareTo(minNodeVersion) <= 0
+            ? CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.get(clusterSettings)
+            : RemoteStoreEnums.PathType.FIXED;
+        RemoteStoreEnums.PathHashAlgorithm pathHashAlgorithm = pathType == RemoteStoreEnums.PathType.FIXED
+            ? null
+            : CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING.get(clusterSettings);
+        Map<String, String> remoteCustomData = new HashMap<>();
+        remoteCustomData.put(RemoteStoreEnums.PathType.NAME, pathType.name());
+        if (Objects.nonNull(pathHashAlgorithm)) {
+            remoteCustomData.put(RemoteStoreEnums.PathHashAlgorithm.NAME, pathHashAlgorithm.name());
+        }
+        return remoteCustomData;
+    }
+
+    /**
+     * Fetches segment and translog repository names from remote store node attributes.
+     * Returns a blank {@link HashMap} if the cluster does not contain any remote nodes.
+     * <br>
+     * Caller need to handle null checks if {@link DiscoveryNodes} object does not have any remote nodes
+     *
+     * @param discoveryNodes Current set of {@link DiscoveryNodes} in the cluster
+     * @return {@link Map} of data repository node attributes keys and their values
+     */
+    public static Map<String, String> getRemoteStoreRepoName(DiscoveryNodes discoveryNodes) {
+        Optional<DiscoveryNode> remoteNode = discoveryNodes.getNodes()
+            .values()
+            .stream()
+            .filter(DiscoveryNode::isRemoteStoreNode)
+            .findFirst();
+        return remoteNode.map(RemoteStoreNodeAttribute::getDataRepoNames).orElseGet(HashMap::new);
+    }
 }
diff --git a/server/src/main/java/org/opensearch/index/search/stats/SearchStats.java b/server/src/main/java/org/opensearch/index/search/stats/SearchStats.java
index 576e00f8f30d1..bb61e1afa05f4 100644
--- a/server/src/main/java/org/opensearch/index/search/stats/SearchStats.java
+++ b/server/src/main/java/org/opensearch/index/search/stats/SearchStats.java
@@ -163,6 +163,8 @@ public static class Stats implements Writeable, ToXContentFragment {
         private long pitTimeInMillis;
         private long pitCurrent;
 
+        private long searchIdleReactivateCount;
+
         @Nullable
         private RequestStatsLongHolder requestStatsLongHolder;
 
@@ -193,7 +195,8 @@ public Stats(
             long pitCurrent,
             long suggestCount,
             long suggestTimeInMillis,
-            long suggestCurrent
+            long suggestCurrent,
+            long searchIdleReactivateCount
         ) {
             this.requestStatsLongHolder = new RequestStatsLongHolder();
             this.queryCount = queryCount;
@@ -220,6 +223,8 @@ public Stats(
             this.pitCount = pitCount;
             this.pitTimeInMillis = pitTimeInMillis;
             this.pitCurrent = pitCurrent;
+
+            this.searchIdleReactivateCount = searchIdleReactivateCount;
         }
 
         private Stats(StreamInput in) throws IOException {
@@ -255,6 +260,10 @@ private Stats(StreamInput in) throws IOException {
                 concurrentQueryCurrent = in.readVLong();
                 queryConcurrency = in.readVLong();
             }
+
+            if (in.getVersion().onOrAfter(Version.V_2_14_0)) {
+                searchIdleReactivateCount = in.readVLong();
+            }
         }
 
         public void add(Stats stats) {
@@ -282,6 +291,8 @@ public void add(Stats stats) {
             pitCount += stats.pitCount;
             pitTimeInMillis += stats.pitTimeInMillis;
             pitCurrent += stats.pitCurrent;
+
+            searchIdleReactivateCount += stats.searchIdleReactivateCount;
         }
 
         public void addForClosingShard(Stats stats) {
@@ -306,6 +317,8 @@ public void addForClosingShard(Stats stats) {
             pitTimeInMillis += stats.pitTimeInMillis;
             pitCurrent += stats.pitCurrent;
             queryConcurrency += stats.queryConcurrency;
+
+            searchIdleReactivateCount += stats.searchIdleReactivateCount;
         }
 
         public long getQueryCount() {
@@ -412,6 +425,10 @@ public long getSuggestCurrent() {
             return suggestCurrent;
         }
 
+        public long getSearchIdleReactivateCount() {
+            return searchIdleReactivateCount;
+        }
+
         public static Stats readStats(StreamInput in) throws IOException {
             return new Stats(in);
         }
@@ -457,6 +474,10 @@ public void writeTo(StreamOutput out) throws IOException {
                 out.writeVLong(concurrentQueryCurrent);
                 out.writeVLong(queryConcurrency);
             }
+
+            if (out.getVersion().onOrAfter(Version.V_2_14_0)) {
+                out.writeVLong(searchIdleReactivateCount);
+            }
         }
 
         @Override
@@ -486,6 +507,8 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.humanReadableField(Fields.SUGGEST_TIME_IN_MILLIS, Fields.SUGGEST_TIME, getSuggestTime());
             builder.field(Fields.SUGGEST_CURRENT, suggestCurrent);
 
+            builder.field(Fields.SEARCH_IDLE_REACTIVATE_COUNT_TOTAL, searchIdleReactivateCount);
+
             if (requestStatsLongHolder != null) {
                 builder.startObject(Fields.REQUEST);
 
@@ -654,6 +677,7 @@ static final class Fields {
         static final String TIME = "time";
         static final String CURRENT = "current";
         static final String TOTAL = "total";
+        static final String SEARCH_IDLE_REACTIVATE_COUNT_TOTAL = "search_idle_reactivate_count_total";
 
     }
 
diff --git a/server/src/main/java/org/opensearch/index/search/stats/ShardSearchStats.java b/server/src/main/java/org/opensearch/index/search/stats/ShardSearchStats.java
index 99e3f8465c5db..3098986852cc1 100644
--- a/server/src/main/java/org/opensearch/index/search/stats/ShardSearchStats.java
+++ b/server/src/main/java/org/opensearch/index/search/stats/ShardSearchStats.java
@@ -213,6 +213,11 @@ public void onFreePitContext(ReaderContext readerContext) {
         totalStats.pitMetric.inc(TimeUnit.NANOSECONDS.toMicros(System.nanoTime() - readerContext.getStartTimeInNano()));
     }
 
+    @Override
+    public void onSearchIdleReactivation() {
+        totalStats.searchIdleMetric.inc();
+    }
+
     /**
      * Holder of statistics values
      *
@@ -239,6 +244,7 @@ static final class StatsHolder {
         final CounterMetric scrollCurrent = new CounterMetric();
         final CounterMetric pitCurrent = new CounterMetric();
         final CounterMetric suggestCurrent = new CounterMetric();
+        final CounterMetric searchIdleMetric = new CounterMetric();
 
         SearchStats.Stats stats() {
             return new SearchStats.Stats(
@@ -260,7 +266,8 @@ SearchStats.Stats stats() {
                 pitCurrent.count(),
                 suggestMetric.count(),
                 TimeUnit.NANOSECONDS.toMillis(suggestMetric.sum()),
-                suggestCurrent.count()
+                suggestCurrent.count(),
+                searchIdleMetric.count()
             );
         }
     }
diff --git a/server/src/main/java/org/opensearch/index/seqno/GlobalCheckpointSyncAction.java b/server/src/main/java/org/opensearch/index/seqno/GlobalCheckpointSyncAction.java
index ca1dfe2d5ad01..c6a1f5f27a875 100644
--- a/server/src/main/java/org/opensearch/index/seqno/GlobalCheckpointSyncAction.java
+++ b/server/src/main/java/org/opensearch/index/seqno/GlobalCheckpointSyncAction.java
@@ -135,7 +135,7 @@ protected void shardOperationOnReplica(Request shardRequest, IndexShard replica,
     private void maybeSyncTranslog(final IndexShard indexShard) throws IOException {
         if (indexShard.getTranslogDurability() == Translog.Durability.REQUEST
             && indexShard.getLastSyncedGlobalCheckpoint() < indexShard.getLastKnownGlobalCheckpoint()
-            && indexShard.isRemoteTranslogEnabled() == false) {
+            && indexShard.indexSettings().isAssignedOnRemoteNode() == false) {
             indexShard.sync();
         }
     }
diff --git a/server/src/main/java/org/opensearch/index/seqno/ReplicationTracker.java b/server/src/main/java/org/opensearch/index/seqno/ReplicationTracker.java
index 0e625e9f30320..6697991aef90b 100644
--- a/server/src/main/java/org/opensearch/index/seqno/ReplicationTracker.java
+++ b/server/src/main/java/org/opensearch/index/seqno/ReplicationTracker.java
@@ -253,6 +253,14 @@ public class ReplicationTracker extends AbstractIndexShardComponent implements L
 
     private volatile ReplicationCheckpoint latestReplicationCheckpoint;
 
+    private final Function<String, Boolean> isShardOnRemoteEnabledNode;
+
+    /**
+     * Flag to indicate whether {@link ReplicationTracker#createMissingPeerRecoveryRetentionLeases(ActionListener)}
+     * has been run successfully
+     */
+    private boolean createdMissingRetentionLeases;
+
     /**
      * Get all retention leases tracked on this shard.
      *
@@ -953,7 +961,13 @@ private boolean invariant() {
             assert checkpoints.get(aId) != null : "aId [" + aId + "] is pending in sync but isn't tracked";
         }
 
-        if (primaryMode && indexSettings.isSoftDeleteEnabled() && hasAllPeerRecoveryRetentionLeases) {
+        if (primaryMode && indexSettings.isSoftDeleteEnabled() && hasAllPeerRecoveryRetentionLeases
+        // Skip assertion if createMissingPeerRecoveryRetentionLeases has not yet run after activating primary context
+        // This is required since during an ongoing remote store migration,
+        // remote enabled primary taking over primary context from another remote enabled shard
+        // might not have retention leases for docrep shard copies
+        // (since all RetentionLease sync actions are blocked on remote shard copies)
+            && createdMissingRetentionLeases) {
             // all tracked shard copies have a corresponding peer-recovery retention lease
             for (final ShardRouting shardRouting : routingTable.assignedShards()) {
                 final CheckpointState cps = checkpoints.get(shardRouting.allocationId().getId());
@@ -999,7 +1013,8 @@ public ReplicationTracker(
         final LongConsumer onGlobalCheckpointUpdated,
         final LongSupplier currentTimeMillisSupplier,
         final BiConsumer<RetentionLeases, ActionListener<ReplicationResponse>> onSyncRetentionLeases,
-        final Supplier<SafeCommitInfo> safeCommitInfoSupplier
+        final Supplier<SafeCommitInfo> safeCommitInfoSupplier,
+        final Function<String, Boolean> isShardOnRemoteEnabledNode
     ) {
         this(
             shardId,
@@ -1011,7 +1026,8 @@ public ReplicationTracker(
             currentTimeMillisSupplier,
             onSyncRetentionLeases,
             safeCommitInfoSupplier,
-            x -> {}
+            x -> {},
+            isShardOnRemoteEnabledNode
         );
     }
 
@@ -1037,7 +1053,8 @@ public ReplicationTracker(
         final LongSupplier currentTimeMillisSupplier,
         final BiConsumer<RetentionLeases, ActionListener<ReplicationResponse>> onSyncRetentionLeases,
         final Supplier<SafeCommitInfo> safeCommitInfoSupplier,
-        final Consumer<ReplicationGroup> onReplicationGroupUpdated
+        final Consumer<ReplicationGroup> onReplicationGroupUpdated,
+        final Function<String, Boolean> isShardOnRemoteEnabledNode
     ) {
         super(shardId, indexSettings);
         assert globalCheckpoint >= SequenceNumbers.UNASSIGNED_SEQ_NO : "illegal initial global checkpoint: " + globalCheckpoint;
@@ -1060,6 +1077,7 @@ public ReplicationTracker(
         this.safeCommitInfoSupplier = safeCommitInfoSupplier;
         this.onReplicationGroupUpdated = onReplicationGroupUpdated;
         this.latestReplicationCheckpoint = indexSettings.isSegRepEnabledOrRemoteNode() ? ReplicationCheckpoint.empty(shardId) : null;
+        this.isShardOnRemoteEnabledNode = isShardOnRemoteEnabledNode;
         assert Version.V_EMPTY.equals(indexSettings.getIndexVersionCreated()) == false;
         assert invariant();
     }
@@ -1088,8 +1106,11 @@ private ReplicationGroup calculateReplicationGroup() {
         } else {
             newVersion = replicationGroup.getVersion() + 1;
         }
-
-        assert indexSettings().isRemoteTranslogStoreEnabled()
+        assert newVersion == 0 || indexSettings.isRemoteTranslogStoreEnabled()
+        // Handle migration cases. Ignore assertion if any of the shard copies in the replication group is assigned to a remote node
+            || replicationGroup.getReplicationTargets()
+                .stream()
+                .anyMatch(shardRouting -> isShardOnRemoteEnabledNode.apply(shardRouting.currentNodeId()))
             || checkpoints.entrySet().stream().filter(e -> e.getValue().tracked).allMatch(e -> e.getValue().replicated)
             : "In absence of remote translog store, all tracked shards must have replication mode as LOGICAL_REPLICATION";
 
@@ -1248,7 +1269,9 @@ private void createReplicationLagTimers() {
                 if (cps.inSync
                     && replicationGroup.getUnavailableInSyncShards().contains(allocationId) == false
                     && isPrimaryRelocation(allocationId) == false
-                    && latestReplicationCheckpoint.isAheadOf(cps.visibleReplicationCheckpoint)) {
+                    && latestReplicationCheckpoint.isAheadOf(cps.visibleReplicationCheckpoint)
+                    && (indexSettings.isSegRepLocalEnabled() == true
+                        || isShardOnRemoteEnabledNode.apply(routingTable.getByAllocationId(allocationId).currentNodeId()))) {
                     cps.checkpointTimers.computeIfAbsent(latestReplicationCheckpoint, ignored -> new SegmentReplicationLagTimer());
                     logger.trace(
                         () -> new ParameterizedMessage(
@@ -1367,7 +1390,7 @@ private void addPeerRecoveryRetentionLeaseForSolePrimary() {
         final String leaseId = getPeerRecoveryRetentionLeaseId(primaryShard);
         if (retentionLeases.get(leaseId) == null) {
             if (replicationGroup.getReplicationTargets().equals(Collections.singletonList(primaryShard))
-                || indexSettings().isRemoteTranslogStoreEnabled()) {
+                || indexSettings.isAssignedOnRemoteNode()) {
                 assert primaryShard.allocationId().getId().equals(shardAllocationId) : routingTable.assignedShards()
                     + " vs "
                     + shardAllocationId;
@@ -1453,7 +1476,12 @@ public synchronized void updateFromClusterManager(
                                 globalCheckpoint,
                                 inSync,
                                 inSync,
-                                isReplicated(initializingId, primaryAllocationId, primaryTargetAllocationId)
+                                isReplicated(
+                                    initializingId,
+                                    primaryAllocationId,
+                                    primaryTargetAllocationId,
+                                    assignedToRemoteStoreNode(routingTable, initializingId)
+                                )
                             )
                         );
                     }
@@ -1472,7 +1500,12 @@ public synchronized void updateFromClusterManager(
                             globalCheckpoint,
                             false,
                             false,
-                            isReplicated(initializingId, primaryAllocationId, primaryTargetAllocationId)
+                            isReplicated(
+                                initializingId,
+                                primaryAllocationId,
+                                primaryTargetAllocationId,
+                                assignedToRemoteStoreNode(routingTable, initializingId)
+                            )
                         )
                     );
                 }
@@ -1486,7 +1519,12 @@ public synchronized void updateFromClusterManager(
                             globalCheckpoint,
                             true,
                             true,
-                            isReplicated(inSyncId, primaryAllocationId, primaryTargetAllocationId)
+                            isReplicated(
+                                inSyncId,
+                                primaryAllocationId,
+                                primaryTargetAllocationId,
+                                assignedToRemoteStoreNode(routingTable, inSyncId)
+                            )
                         )
                     );
                 }
@@ -1503,6 +1541,12 @@ public synchronized void updateFromClusterManager(
         assert invariant();
     }
 
+    private boolean assignedToRemoteStoreNode(IndexShardRoutingTable routingTable, String allocationId) {
+        return indexSettings().isRemoteStoreEnabled()
+            || (routingTable.getByAllocationId(allocationId) != null
+                && isShardOnRemoteEnabledNode.apply(routingTable.getByAllocationId(allocationId).currentNodeId()));
+    }
+
     /**
      * Returns whether the requests are replicated considering the remote translog existence, current/primary/primary target allocation ids.
      *
@@ -1511,13 +1555,16 @@ public synchronized void updateFromClusterManager(
      * @param primaryTargetAllocationId primary target allocation id
      * @return the replication mode.
      */
-    private boolean isReplicated(String allocationId, String primaryAllocationId, String primaryTargetAllocationId) {
-        // If remote translog is enabled, then returns replication mode checking current allocation id against the
+    private boolean isReplicated(
+        String allocationId,
+        String primaryAllocationId,
+        String primaryTargetAllocationId,
+        boolean assignedToRemoteStoreNode
+    ) {
+        // If assigned to a remote node, returns true if given allocation id matches the primary or it's relocation target allocation
         // primary and primary target allocation id.
-        // If remote translog is enabled, then returns true if given allocation id matches the primary or it's relocation target allocation
-        // id.
-        if (indexSettings().isRemoteTranslogStoreEnabled()) {
-            return (allocationId.equals(primaryAllocationId) || allocationId.equals(primaryTargetAllocationId));
+        if (assignedToRemoteStoreNode == true) {
+            return allocationId.equals(primaryAllocationId) || allocationId.equals(primaryTargetAllocationId);
         }
         // For other case which is local translog, return true as the requests are replicated to all shards in the replication group.
         return true;
@@ -1807,19 +1854,34 @@ private synchronized void setHasAllPeerRecoveryRetentionLeases() {
         assert invariant();
     }
 
+    private synchronized void setCreatedMissingRetentionLeases() {
+        createdMissingRetentionLeases = true;
+        assert invariant();
+    }
+
     public synchronized boolean hasAllPeerRecoveryRetentionLeases() {
         return hasAllPeerRecoveryRetentionLeases;
     }
 
     /**
-     * Create any required peer-recovery retention leases that do not currently exist because we just did a rolling upgrade from a version
-     * prior to {@code LegacyESVersion#V_7_4_0} that does not create peer-recovery retention leases.
+     * Create any required peer-recovery retention leases that do not currently exist. This can happen if either:
+     * - We just did a rolling upgrade from a version prior to {@code LegacyESVersion#V_7_4_0} that does not create peer-recovery retention leases.
+     * - In a mixed mode cluster (during remote store migration), a remote enabled primary shard copy fails over to another remote enabled shard copy,
+     * but the replication group still has other shards in docrep nodes
      */
     public synchronized void createMissingPeerRecoveryRetentionLeases(ActionListener<Void> listener) {
-        if (hasAllPeerRecoveryRetentionLeases == false) {
+        // Create missing RetentionLeases if the primary is on a remote enabled
+        // and the replication group has at-least one shard copy in docrep enabled node
+        // No-Op if retention leases for the tracked shard copy already exists
+        boolean createMissingRetentionLeasesDuringMigration = indexSettings.isAssignedOnRemoteNode()
+            && replicationGroup.getReplicationTargets()
+                .stream()
+                .anyMatch(shardRouting -> isShardOnRemoteEnabledNode.apply(shardRouting.currentNodeId()) == false);
+        if (hasAllPeerRecoveryRetentionLeases == false || createMissingRetentionLeasesDuringMigration) {
             final List<ShardRouting> shardRoutings = routingTable.assignedShards();
             final GroupedActionListener<ReplicationResponse> groupedActionListener = new GroupedActionListener<>(ActionListener.wrap(vs -> {
                 setHasAllPeerRecoveryRetentionLeases();
+                setCreatedMissingRetentionLeases();
                 listener.onResponse(null);
             }, listener::onFailure), shardRoutings.size());
             for (ShardRouting shardRouting : shardRoutings) {
diff --git a/server/src/main/java/org/opensearch/index/shard/CheckpointRefreshListener.java b/server/src/main/java/org/opensearch/index/shard/CheckpointRefreshListener.java
index 675d60ec2b63d..7f0806059155a 100644
--- a/server/src/main/java/org/opensearch/index/shard/CheckpointRefreshListener.java
+++ b/server/src/main/java/org/opensearch/index/shard/CheckpointRefreshListener.java
@@ -43,7 +43,7 @@ protected boolean performAfterRefreshWithPermit(boolean didRefresh) {
         if (didRefresh
             && shard.state() == IndexShardState.STARTED
             && shard.getReplicationTracker().isPrimaryMode()
-            && !shard.indexSettings.isSegRepWithRemoteEnabled()) {
+            && shard.indexSettings.isAssignedOnRemoteNode() == false) {
             publisher.publish(shard, shard.getLatestReplicationCheckpoint());
         }
         return true;
diff --git a/server/src/main/java/org/opensearch/index/shard/IndexShard.java b/server/src/main/java/org/opensearch/index/shard/IndexShard.java
index 72ce858661031..18d4a2ca6d639 100644
--- a/server/src/main/java/org/opensearch/index/shard/IndexShard.java
+++ b/server/src/main/java/org/opensearch/index/shard/IndexShard.java
@@ -67,6 +67,8 @@
 import org.opensearch.cluster.metadata.DataStream;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.metadata.MappingMetadata;
+import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.cluster.node.DiscoveryNodes;
 import org.opensearch.cluster.routing.IndexShardRoutingTable;
 import org.opensearch.cluster.routing.RecoverySource;
 import org.opensearch.cluster.routing.RecoverySource.SnapshotRecoverySource;
@@ -179,6 +181,7 @@
 import org.opensearch.index.warmer.WarmerStats;
 import org.opensearch.indices.IndexingMemoryController;
 import org.opensearch.indices.IndicesService;
+import org.opensearch.indices.RemoteStoreSettings;
 import org.opensearch.indices.cluster.IndicesClusterStateService;
 import org.opensearch.indices.recovery.PeerRecoveryTargetService;
 import org.opensearch.indices.recovery.RecoveryFailedException;
@@ -349,12 +352,14 @@ Runnable getGlobalCheckpointSyncer() {
     private final List<ReferenceManager.RefreshListener> internalRefreshListener = new ArrayList<>();
     private final RemoteStoreFileDownloader fileDownloader;
     private final RecoverySettings recoverySettings;
+    private final RemoteStoreSettings remoteStoreSettings;
     /*
      On source doc rep node,  It will be DOCREP_NON_MIGRATING.
      On source remote node , it will be REMOTE_MIGRATING_SEEDED when relocating from remote node
      On source remote node , it will be REMOTE_MIGRATING_UNSEEDED when relocating from docrep node
      */
     private final ShardMigrationState shardMigrationState;
+    private DiscoveryNodes discoveryNodes;
 
     public IndexShard(
         final ShardRouting shardRouting,
@@ -381,10 +386,11 @@ public IndexShard(
         @Nullable final SegmentReplicationCheckpointPublisher checkpointPublisher,
         @Nullable final Store remoteStore,
         final RemoteStoreStatsTrackerFactory remoteStoreStatsTrackerFactory,
-        final Supplier<TimeValue> clusterRemoteTranslogBufferIntervalSupplier,
         final String nodeId,
         final RecoverySettings recoverySettings,
-        boolean seedRemote
+        final RemoteStoreSettings remoteStoreSettings,
+        boolean seedRemote,
+        final DiscoveryNodes discoveryNodes
     ) throws IOException {
         super(shardRouting.shardId(), indexSettings);
         assert shardRouting.initializing();
@@ -404,8 +410,8 @@ public IndexShard(
             logger,
             threadPool,
             this::getEngine,
-            indexSettings.isRemoteNode(),
-            () -> getRemoteTranslogUploadBufferInterval(clusterRemoteTranslogBufferIntervalSupplier)
+            indexSettings.isAssignedOnRemoteNode(),
+            () -> getRemoteTranslogUploadBufferInterval(remoteStoreSettings::getClusterRemoteTranslogBufferInterval)
         );
         this.mapperService = mapperService;
         this.indexCache = indexCache;
@@ -430,7 +436,7 @@ public IndexShard(
         logger.debug("state: [CREATED]");
 
         this.checkIndexOnStartup = indexSettings.getValue(IndexSettings.INDEX_CHECK_ON_STARTUP);
-        this.translogConfig = new TranslogConfig(shardId, shardPath().resolveTranslog(), indexSettings, bigArrays, nodeId);
+        this.translogConfig = new TranslogConfig(shardId, shardPath().resolveTranslog(), indexSettings, bigArrays, nodeId, seedRemote);
         final String aId = shardRouting.allocationId().getId();
         final long primaryTerm = indexSettings.getIndexMetadata().primaryTerm(shardId.id());
         this.pendingPrimaryTerm = primaryTerm;
@@ -446,7 +452,8 @@ public IndexShard(
             threadPool::absoluteTimeInMillis,
             (retentionLeases, listener) -> retentionLeaseSyncer.sync(shardId, aId, getPendingPrimaryTerm(), retentionLeases, listener),
             this::getSafeCommitInfo,
-            pendingReplicationActions
+            pendingReplicationActions,
+            isShardOnRemoteEnabledNode
         );
 
         // the query cache is a node-level thing, however we want the most popular filters
@@ -481,8 +488,10 @@ public boolean shouldCache(Query query) {
             : mapperService.documentMapper().mappers().containsTimeStampField();
         this.remoteStoreStatsTrackerFactory = remoteStoreStatsTrackerFactory;
         this.recoverySettings = recoverySettings;
+        this.remoteStoreSettings = remoteStoreSettings;
         this.fileDownloader = new RemoteStoreFileDownloader(shardRouting.shardId(), threadPool, recoverySettings);
         this.shardMigrationState = getShardMigrationState(indexSettings, seedRemote);
+        this.discoveryNodes = discoveryNodes;
     }
 
     public ThreadPool getThreadPool() {
@@ -503,6 +512,23 @@ public boolean shouldSeedRemoteStore() {
         return shardMigrationState == REMOTE_MIGRATING_UNSEEDED;
     }
 
+    /**
+     * To be delegated to {@link ReplicationTracker} so that relevant remote store based
+     * operations can be ignored during engine migration
+     * <p>
+     * Has explicit null checks to ensure that the {@link ReplicationTracker#invariant()}
+     * checks does not fail during a cluster manager state update when the latest replication group
+     * calculation is not yet done and the cached replication group details are available
+     */
+    public Function<String, Boolean> isShardOnRemoteEnabledNode = nodeId -> {
+        DiscoveryNode node = discoveryNodes.get(nodeId);
+        if (node != null) {
+            logger.trace("Node {} has remote_enabled as {}", nodeId, node.isRemoteStoreNode());
+            return node.isRemoteStoreNode();
+        }
+        return false;
+    };
+
     public boolean isRemoteSeeded() {
         return shardMigrationState == REMOTE_MIGRATING_SEEDED;
     }
@@ -598,6 +624,10 @@ public RecoverySettings getRecoverySettings() {
         return recoverySettings;
     }
 
+    public RemoteStoreSettings getRemoteStoreSettings() {
+        return remoteStoreSettings;
+    }
+
     public RemoteStoreFileDownloader getFileDownloader() {
         return fileDownloader;
     }
@@ -609,8 +639,10 @@ public void updateShardState(
         final BiConsumer<IndexShard, ActionListener<ResyncTask>> primaryReplicaSyncer,
         final long applyingClusterStateVersion,
         final Set<String> inSyncAllocationIds,
-        final IndexShardRoutingTable routingTable
+        final IndexShardRoutingTable routingTable,
+        DiscoveryNodes discoveryNodes
     ) throws IOException {
+        this.discoveryNodes = discoveryNodes;
         final ShardRouting currentRouting;
         synchronized (mutex) {
             currentRouting = this.shardRouting;
@@ -1462,7 +1494,7 @@ public SegmentsStats segmentStats(boolean includeSegmentFileSizes, boolean inclu
         SegmentsStats segmentsStats = getEngine().segmentsStats(includeSegmentFileSizes, includeUnloadedSegments);
         segmentsStats.addBitsetMemoryInBytes(shardBitsetFilterCache.getMemorySizeInBytes());
         // Populate remote_store stats only if the index is remote store backed
-        if (indexSettings().isRemoteNode()) {
+        if (indexSettings().isAssignedOnRemoteNode()) {
             segmentsStats.addRemoteSegmentStats(
                 new RemoteSegmentStats(remoteStoreStatsTrackerFactory.getRemoteSegmentTransferTracker(shardId).stats())
             );
@@ -1484,7 +1516,7 @@ public FieldDataStats fieldDataStats(String... fields) {
     public TranslogStats translogStats() {
         TranslogStats translogStats = getEngine().translogManager().getTranslogStats();
         // Populate remote_store stats only if the index is remote store backed
-        if (indexSettings.isRemoteNode()) {
+        if (indexSettings.isAssignedOnRemoteNode()) {
             translogStats.addRemoteTranslogStats(
                 new RemoteTranslogStats(remoteStoreStatsTrackerFactory.getRemoteTranslogTransferTracker(shardId).stats())
             );
@@ -1523,7 +1555,7 @@ public void flush(FlushRequest request) {
      * {@link org.opensearch.index.translog.TranslogDeletionPolicy} for details
      */
     public void trimTranslog() {
-        if (indexSettings.isRemoteNode()) {
+        if (indexSettings.isAssignedOnRemoteNode()) {
             return;
         }
         verifyNotClosed();
@@ -2043,7 +2075,7 @@ public void close(String reason, boolean flushEngine, boolean deleted) throws IO
     ToDo : Fix this https://github.com/opensearch-project/OpenSearch/issues/8003
      */
     public RemoteSegmentStoreDirectory getRemoteDirectory() {
-        assert indexSettings.isRemoteNode();
+        assert indexSettings.isAssignedOnRemoteNode();
         assert remoteStore.directory() instanceof FilterDirectory : "Store.directory is not an instance of FilterDirectory";
         FilterDirectory remoteStoreDirectory = (FilterDirectory) remoteStore.directory();
         FilterDirectory byteSizeCachingStoreDirectory = (FilterDirectory) remoteStoreDirectory.getDelegate();
@@ -2056,7 +2088,7 @@ public RemoteSegmentStoreDirectory getRemoteDirectory() {
      * is in sync with local
      */
     public boolean isRemoteSegmentStoreInSync() {
-        assert indexSettings.isRemoteNode();
+        assert indexSettings.isAssignedOnRemoteNode();
         try {
             RemoteSegmentStoreDirectory directory = getRemoteDirectory();
             if (directory.readLatestMetadataFile() != null) {
@@ -2086,16 +2118,17 @@ public boolean isRemoteSegmentStoreInSync() {
         return false;
     }
 
-    public void waitForRemoteStoreSync() {
+    public void waitForRemoteStoreSync() throws IOException {
         waitForRemoteStoreSync(() -> {});
     }
 
     /*
     Blocks the calling thread,  waiting for the remote store to get synced till internal Remote Upload Timeout
     Calls onProgress on seeing an increased file count on remote
+    Throws IOException if the remote store is not synced within the timeout
     */
-    public void waitForRemoteStoreSync(Runnable onProgress) {
-        assert indexSettings.isRemoteNode();
+    public void waitForRemoteStoreSync(Runnable onProgress) throws IOException {
+        assert indexSettings.isAssignedOnRemoteNode();
         RemoteSegmentStoreDirectory directory = getRemoteDirectory();
         int segmentUploadeCount = 0;
         if (shardRouting.primary() == false) {
@@ -2106,7 +2139,7 @@ public void waitForRemoteStoreSync(Runnable onProgress) {
         while (System.nanoTime() - startNanos < getRecoverySettings().internalRemoteUploadTimeout().nanos()) {
             try {
                 if (isRemoteSegmentStoreInSync()) {
-                    break;
+                    return;
                 } else {
                     if (directory.getSegmentsUploadedToRemoteStore().size() > segmentUploadeCount) {
                         onProgress.run();
@@ -2124,6 +2157,11 @@ public void waitForRemoteStoreSync(Runnable onProgress) {
                 return;
             }
         }
+        throw new IOException(
+            "Failed to upload to remote segment store within remote upload timeout of "
+                + getRecoverySettings().internalRemoteUploadTimeout().getMinutes()
+                + " minutes"
+        );
     }
 
     public void preRecovery() {
@@ -2270,7 +2308,7 @@ public long recoverLocallyAndFetchStartSeqNo(boolean localTranslog) {
      * @return the starting sequence number from which the recovery should start.
      */
     private long recoverLocallyUptoLastCommit() {
-        assert indexSettings.isRemoteNode() : "Remote translog store is not enabled";
+        assert indexSettings.isAssignedOnRemoteNode() : "Remote translog store is not enabled";
         long seqNo;
         validateLocalRecoveryState();
 
@@ -2879,7 +2917,7 @@ public void restoreFromRepository(Repository repository, ActionListener<Boolean>
      *
      * @return {@code true} if the engine should be flushed
      */
-    boolean shouldPeriodicallyFlush() {
+    public boolean shouldPeriodicallyFlush() {
         final Engine engine = getEngineOrNull();
         if (engine != null) {
             try {
@@ -3489,7 +3527,7 @@ public void updateGlobalCheckpointOnReplica(final long globalCheckpoint, final S
              * update local checkpoint at replica, so the local checkpoint at replica can be less than globalCheckpoint.
              */
             assert (state() != IndexShardState.POST_RECOVERY && state() != IndexShardState.STARTED)
-                || indexSettings.isRemoteTranslogStoreEnabled() : "supposedly in-sync shard copy received a global checkpoint ["
+                || indexSettings.isAssignedOnRemoteNode() : "supposedly in-sync shard copy received a global checkpoint ["
                     + globalCheckpoint
                     + "] "
                     + "that is higher than its local checkpoint ["
@@ -3533,7 +3571,7 @@ assert getLocalCheckpoint() == primaryContext.getCheckpointStates().get(allocati
     }
 
     private void postActivatePrimaryMode() {
-        if (indexSettings.isRemoteNode()) {
+        if (indexSettings.isAssignedOnRemoteNode()) {
             // We make sure to upload translog (even if it does not contain any operations) to remote translog.
             // This helps to get a consistent state in remote store where both remote segment store and remote
             // translog contains data.
@@ -3986,7 +4024,7 @@ private boolean isRemoteStoreEnabled() {
     }
 
     public boolean isRemoteTranslogEnabled() {
-        return indexSettings() != null && indexSettings().isRemoteTranslogStoreEnabled();
+        return indexSettings() != null && (indexSettings().isRemoteTranslogStoreEnabled());
     }
 
     /**
@@ -4003,7 +4041,7 @@ public boolean enableUploadToRemoteTranslog() {
     }
 
     private boolean hasOneRemoteSegmentSyncHappened() {
-        assert indexSettings.isRemoteNode();
+        assert indexSettings.isAssignedOnRemoteNode();
         // We upload remote translog only after one remote segment upload in case of migration
         RemoteSegmentStoreDirectory rd = getRemoteDirectory();
         AtomicBoolean segment_n_uploaded = new AtomicBoolean(false);
@@ -4455,6 +4493,7 @@ public Durability getTranslogDurability() {
     /**
      * Schedules a flush or translog generation roll if needed but will not schedule more than one concurrently. The operation will be
      * executed asynchronously on the flush thread pool.
+     * Can also schedule a flush if decided by translog manager
      */
     public void afterWriteOperation() {
         if (shouldPeriodicallyFlush() || shouldRollTranslogGeneration()) {
@@ -4617,7 +4656,7 @@ public final boolean isSearchIdle() {
     public final boolean isSearchIdleSupported() {
         // If the index is remote store backed, then search idle is not supported. This is to ensure that async refresh
         // task continues to upload to remote store periodically.
-        if (isRemoteTranslogEnabled() || indexSettings.isRemoteNode()) {
+        if (isRemoteTranslogEnabled() || indexSettings.isAssignedOnRemoteNode()) {
             return false;
         }
         return indexSettings.isSegRepEnabledOrRemoteNode() == false || indexSettings.getNumberOfReplicas() == 0;
@@ -4683,9 +4722,14 @@ public void afterRefresh(boolean didRefresh) {
      *                 <code>true</code> if the listener was registered to wait for a refresh.
      */
     public final void awaitShardSearchActive(Consumer<Boolean> listener) {
+        boolean isSearchIdle = isSearchIdle();
         markSearcherAccessed(); // move the shard into non-search idle
         final Translog.Location location = pendingRefreshLocation.get();
         if (location != null) {
+            if (isSearchIdle) {
+                SearchOperationListener searchOperationListener = getSearchOperationListener();
+                searchOperationListener.onSearchIdleReactivation();
+            }
             addRefreshListener(location, (b) -> {
                 pendingRefreshLocation.compareAndSet(location, null);
                 listener.accept(true);
@@ -4932,7 +4976,7 @@ public void deleteTranslogFilesFromRemoteTranslog() throws IOException {
         TranslogFactory translogFactory = translogFactorySupplier.apply(indexSettings, shardRouting);
         assert translogFactory instanceof RemoteBlobStoreInternalTranslogFactory;
         Repository repository = ((RemoteBlobStoreInternalTranslogFactory) translogFactory).getRepository();
-        RemoteFsTranslog.cleanup(repository, shardId, getThreadPool());
+        RemoteFsTranslog.cleanup(repository, shardId, getThreadPool(), indexSettings.getRemoteStorePathStrategy(), remoteStoreSettings);
     }
 
     /*
@@ -4949,7 +4993,16 @@ public void syncTranslogFilesFromRemoteTranslog() throws IOException {
         TranslogFactory translogFactory = translogFactorySupplier.apply(indexSettings, shardRouting);
         assert translogFactory instanceof RemoteBlobStoreInternalTranslogFactory;
         Repository repository = ((RemoteBlobStoreInternalTranslogFactory) translogFactory).getRepository();
-        RemoteFsTranslog.download(repository, shardId, getThreadPool(), shardPath().resolveTranslog(), logger);
+        RemoteFsTranslog.download(
+            repository,
+            shardId,
+            getThreadPool(),
+            shardPath().resolveTranslog(),
+            indexSettings.getRemoteStorePathStrategy(),
+            remoteStoreSettings,
+            logger,
+            shouldSeedRemoteStore()
+        );
     }
 
     /**
@@ -5244,9 +5297,9 @@ enum ShardMigrationState {
     }
 
     static ShardMigrationState getShardMigrationState(IndexSettings indexSettings, boolean shouldSeed) {
-        if (indexSettings.isRemoteNode() && indexSettings.isRemoteStoreEnabled()) {
+        if (indexSettings.isAssignedOnRemoteNode() && indexSettings.isRemoteStoreEnabled()) {
             return REMOTE_NON_MIGRATING;
-        } else if (indexSettings.isRemoteNode()) {
+        } else if (indexSettings.isAssignedOnRemoteNode()) {
             return shouldSeed ? REMOTE_MIGRATING_UNSEEDED : REMOTE_MIGRATING_SEEDED;
         }
         return ShardMigrationState.DOCREP_NON_MIGRATING;
diff --git a/server/src/main/java/org/opensearch/index/shard/ReleasableRetryableRefreshListener.java b/server/src/main/java/org/opensearch/index/shard/ReleasableRetryableRefreshListener.java
index 757275932c5f1..80daefc4482fc 100644
--- a/server/src/main/java/org/opensearch/index/shard/ReleasableRetryableRefreshListener.java
+++ b/server/src/main/java/org/opensearch/index/shard/ReleasableRetryableRefreshListener.java
@@ -13,6 +13,7 @@
 import org.opensearch.common.lease.Releasable;
 import org.opensearch.common.lease.Releasables;
 import org.opensearch.common.unit.TimeValue;
+import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException;
 import org.opensearch.threadpool.ThreadPool;
 
 import java.io.IOException;
@@ -129,6 +130,12 @@ private void scheduleRetry(TimeValue interval, String retryThreadPoolName, boole
             );
             scheduled = true;
             getLogger().info("Scheduled retry with didRefresh={}", didRefresh);
+        } catch (OpenSearchRejectedExecutionException e) {
+            if (e.isExecutorShutdown()) {
+                getLogger().info("Scheduling retry with didRefresh={} failed due to executor shut down", didRefresh);
+            } else {
+                throw e;
+            }
         } finally {
             if (scheduled == false) {
                 retryScheduled.set(false);
diff --git a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java
index fb96102bc6094..bfb841307af49 100644
--- a/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java
+++ b/server/src/main/java/org/opensearch/index/shard/RemoteStoreRefreshListener.java
@@ -224,7 +224,7 @@ private boolean syncSegments() {
                 // is considered as a first refresh post commit. A cleanup of stale commit files is triggered.
                 // This is done to avoid delete post each refresh.
                 if (isRefreshAfterCommit()) {
-                    remoteDirectory.deleteStaleSegmentsAsync(indexShard.getRecoverySettings().getMinRemoteSegmentMetadataFiles());
+                    remoteDirectory.deleteStaleSegmentsAsync(indexShard.getRemoteStoreSettings().getMinRemoteSegmentMetadataFiles());
                 }
 
                 try (GatedCloseable<SegmentInfos> segmentInfosGatedCloseable = indexShard.getSegmentInfosSnapshot()) {
@@ -437,10 +437,14 @@ private void uploadNewSegments(
                 batchUploadListener.onFailure(ex);
             });
             statsListener.beforeUpload(src);
-            remoteDirectory.copyFrom(storeDirectory, src, IOContext.DEFAULT, aggregatedListener);
+            remoteDirectory.copyFrom(storeDirectory, src, IOContext.DEFAULT, aggregatedListener, isLowPriorityUpload());
         }
     }
 
+    private boolean isLowPriorityUpload() {
+        return isLocalOrSnapshotRecovery();
+    }
+
     /**
      * Whether to upload a file or not depending on whether file is in excluded list or has been already uploaded.
      *
diff --git a/server/src/main/java/org/opensearch/index/shard/SearchOperationListener.java b/server/src/main/java/org/opensearch/index/shard/SearchOperationListener.java
index 849a4f9c15318..94079db468f9c 100644
--- a/server/src/main/java/org/opensearch/index/shard/SearchOperationListener.java
+++ b/server/src/main/java/org/opensearch/index/shard/SearchOperationListener.java
@@ -145,6 +145,11 @@ default void onNewPitContext(ReaderContext readerContext) {}
      */
     default void onFreePitContext(ReaderContext readerContext) {}
 
+    /**
+     * Executed when a shard goes from idle to non-idle state
+     */
+    default void onSearchIdleReactivation() {}
+
     /**
      * A Composite listener that multiplexes calls to each of the listeners methods.
      */
@@ -310,5 +315,16 @@ public void onFreePitContext(ReaderContext readerContext) {
                 }
             }
         }
+
+        @Override
+        public void onSearchIdleReactivation() {
+            for (SearchOperationListener listener : listeners) {
+                try {
+                    listener.onSearchIdleReactivation();
+                } catch (Exception e) {
+                    logger.warn(() -> new ParameterizedMessage("onNewSearchIdleReactivation listener [{}] failed", listener), e);
+                }
+            }
+        }
     }
 }
diff --git a/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java b/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java
index 5f09b1a0802f3..8d689e8769728 100644
--- a/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java
+++ b/server/src/main/java/org/opensearch/index/shard/StoreRecovery.java
@@ -193,13 +193,6 @@ void recoverFromLocalShards(
                     indexShard.getEngine().forceMerge(false, -1, false, false, false, UUIDs.randomBase64UUID());
                     if (indexShard.isRemoteTranslogEnabled() && indexShard.shardRouting.primary()) {
                         indexShard.waitForRemoteStoreSync();
-                        if (indexShard.isRemoteSegmentStoreInSync() == false) {
-                            throw new IndexShardRecoveryException(
-                                indexShard.shardId(),
-                                "failed to upload to remote",
-                                new IOException("Failed to upload to remote segment store")
-                            );
-                        }
                     }
                     return true;
                 } catch (IOException ex) {
@@ -409,7 +402,8 @@ void recoverFromSnapshotAndRemoteStore(
                 RemoteSegmentStoreDirectory sourceRemoteDirectory = (RemoteSegmentStoreDirectory) directoryFactory.newDirectory(
                     remoteStoreRepository,
                     indexUUID,
-                    shardId
+                    shardId,
+                    shallowCopyShardMetadata.getRemoteStorePathStrategy()
                 );
                 sourceRemoteDirectory.initializeToSpecificCommit(
                     primaryTerm,
@@ -435,10 +429,6 @@ void recoverFromSnapshotAndRemoteStore(
                 indexShard.finalizeRecovery();
                 if (indexShard.isRemoteTranslogEnabled() && indexShard.shardRouting.primary()) {
                     indexShard.waitForRemoteStoreSync();
-                    if (indexShard.isRemoteSegmentStoreInSync() == false) {
-                        listener.onFailure(new IndexShardRestoreFailedException(shardId, "Failed to upload to remote segment store"));
-                        return;
-                    }
                 }
                 indexShard.postRecovery("restore done");
 
@@ -660,6 +650,14 @@ private void internalRecoverFromStore(IndexShard indexShard) throws IndexShardRe
                 indexShard.recoveryState().getIndex().setFileDetailsComplete();
             }
             indexShard.openEngineAndRecoverFromTranslog();
+            if (indexShard.shouldSeedRemoteStore()) {
+                indexShard.getThreadPool().executor(ThreadPool.Names.GENERIC).execute(() -> {
+                    logger.info("Attempting to seed Remote Store via local recovery for {}", indexShard.shardId());
+                    indexShard.refresh("remote store migration");
+                });
+                indexShard.waitForRemoteStoreSync();
+                logger.info("Remote Store is now seeded via local recovery for {}", indexShard.shardId());
+            }
             indexShard.getEngine().fillSeqNoGaps(indexShard.getPendingPrimaryTerm());
             indexShard.finalizeRecovery();
             indexShard.postRecovery("post recovery from shard_store");
@@ -721,10 +719,6 @@ private void restore(
             indexShard.finalizeRecovery();
             if (indexShard.isRemoteTranslogEnabled() && indexShard.shardRouting.primary()) {
                 indexShard.waitForRemoteStoreSync();
-                if (indexShard.isRemoteSegmentStoreInSync() == false) {
-                    listener.onFailure(new IndexShardRestoreFailedException(shardId, "Failed to upload to remote segment store"));
-                    return;
-                }
             }
             indexShard.postRecovery("restore done");
             listener.onResponse(true);
diff --git a/server/src/main/java/org/opensearch/index/snapshots/blobstore/RemoteStoreShardShallowCopySnapshot.java b/server/src/main/java/org/opensearch/index/snapshots/blobstore/RemoteStoreShardShallowCopySnapshot.java
index d54e9686ab951..9c0ea42810e16 100644
--- a/server/src/main/java/org/opensearch/index/snapshots/blobstore/RemoteStoreShardShallowCopySnapshot.java
+++ b/server/src/main/java/org/opensearch/index/snapshots/blobstore/RemoteStoreShardShallowCopySnapshot.java
@@ -8,17 +8,22 @@
 
 package org.opensearch.index.snapshots.blobstore;
 
+import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.opensearch.OpenSearchParseException;
 import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.ParseField;
 import org.opensearch.core.xcontent.ToXContentFragment;
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.core.xcontent.XContentParser;
+import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm;
+import org.opensearch.index.remote.RemoteStoreEnums.PathType;
+import org.opensearch.index.remote.RemoteStorePathStrategy;
 import org.opensearch.index.snapshots.IndexShardSnapshotStatus;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Objects;
 
 /**
  * Remote Store based Shard snapshot metadata
@@ -41,8 +46,10 @@ public class RemoteStoreShardShallowCopySnapshot implements ToXContentFragment,
     private final String repositoryBasePath;
     private final String indexUUID;
     private final List<String> fileNames;
+    private final PathType pathType;
+    private final PathHashAlgorithm pathHashAlgorithm;
 
-    static final String DEFAULT_VERSION = "1";
+    static final String DEFAULT_VERSION = "2";
     static final String NAME = "name";
     static final String VERSION = "version";
     static final String INDEX_VERSION = "index_version";
@@ -61,6 +68,8 @@ public class RemoteStoreShardShallowCopySnapshot implements ToXContentFragment,
 
     static final String TOTAL_FILE_COUNT = "number_of_files";
     static final String TOTAL_SIZE = "total_size";
+    static final String PATH_TYPE = "path_type";
+    static final String PATH_HASH_ALGORITHM = "path_hash_algorithm";
 
     private static final ParseField PARSE_NAME = new ParseField(NAME);
     private static final ParseField PARSE_VERSION = new ParseField(VERSION);
@@ -75,6 +84,8 @@ public class RemoteStoreShardShallowCopySnapshot implements ToXContentFragment,
     private static final ParseField PARSE_REMOTE_STORE_REPOSITORY = new ParseField(REMOTE_STORE_REPOSITORY);
     private static final ParseField PARSE_REPOSITORY_BASE_PATH = new ParseField(REPOSITORY_BASE_PATH);
     private static final ParseField PARSE_FILE_NAMES = new ParseField(FILE_NAMES);
+    private static final ParseField PARSE_PATH_TYPE = new ParseField(PATH_TYPE);
+    private static final ParseField PARSE_PATH_HASH_ALGORITHM = new ParseField(PATH_HASH_ALGORITHM);
 
     /**
      * Serializes shard snapshot metadata info into JSON
@@ -101,6 +112,14 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws
             builder.value(fileName);
         }
         builder.endArray();
+        // We are handling NP check since a cluster can have indexes created earlier which do not have remote store
+        // path type and path hash algorithm in its custom data in index metadata.
+        if (Objects.nonNull(pathType)) {
+            builder.field(PATH_TYPE, pathType.getCode());
+        }
+        if (Objects.nonNull(pathHashAlgorithm)) {
+            builder.field(PATH_HASH_ALGORITHM, pathHashAlgorithm.getCode());
+        }
         return builder;
     }
 
@@ -116,34 +135,30 @@ public RemoteStoreShardShallowCopySnapshot(
         String indexUUID,
         String remoteStoreRepository,
         String repositoryBasePath,
-        List<String> fileNames
+        List<String> fileNames,
+        PathType pathType,
+        PathHashAlgorithm pathHashAlgorithm
     ) {
-        this.version = DEFAULT_VERSION;
-        verifyParameters(
-            version,
+        this(
+            DEFAULT_VERSION,
             snapshot,
             indexVersion,
             primaryTerm,
             commitGeneration,
+            startTime,
+            time,
+            totalFileCount,
+            totalSize,
             indexUUID,
             remoteStoreRepository,
-            repositoryBasePath
+            repositoryBasePath,
+            fileNames,
+            pathType,
+            pathHashAlgorithm
         );
-        this.snapshot = snapshot;
-        this.indexVersion = indexVersion;
-        this.primaryTerm = primaryTerm;
-        this.commitGeneration = commitGeneration;
-        this.startTime = startTime;
-        this.time = time;
-        this.totalFileCount = totalFileCount;
-        this.totalSize = totalSize;
-        this.indexUUID = indexUUID;
-        this.remoteStoreRepository = remoteStoreRepository;
-        this.repositoryBasePath = repositoryBasePath;
-        this.fileNames = fileNames;
     }
 
-    private RemoteStoreShardShallowCopySnapshot(
+    RemoteStoreShardShallowCopySnapshot(
         String version,
         String snapshot,
         long indexVersion,
@@ -156,7 +171,9 @@ private RemoteStoreShardShallowCopySnapshot(
         String indexUUID,
         String remoteStoreRepository,
         String repositoryBasePath,
-        List<String> fileNames
+        List<String> fileNames,
+        PathType pathType,
+        PathHashAlgorithm pathHashAlgorithm
     ) {
         verifyParameters(
             version,
@@ -166,7 +183,9 @@ private RemoteStoreShardShallowCopySnapshot(
             commitGeneration,
             indexUUID,
             remoteStoreRepository,
-            repositoryBasePath
+            repositoryBasePath,
+            pathType,
+            pathHashAlgorithm
         );
         this.version = version;
         this.snapshot = snapshot;
@@ -181,6 +200,8 @@ private RemoteStoreShardShallowCopySnapshot(
         this.remoteStoreRepository = remoteStoreRepository;
         this.repositoryBasePath = repositoryBasePath;
         this.fileNames = fileNames;
+        this.pathType = pathType;
+        this.pathHashAlgorithm = pathHashAlgorithm;
     }
 
     /**
@@ -203,6 +224,8 @@ public static RemoteStoreShardShallowCopySnapshot fromXContent(XContentParser pa
         long primaryTerm = -1;
         long commitGeneration = -1;
         List<String> fileNames = new ArrayList<>();
+        PathType pathType = null;
+        PathHashAlgorithm pathHashAlgorithm = null;
 
         if (parser.currentToken() == null) { // fresh parser? move to the first token
             parser.nextToken();
@@ -237,6 +260,10 @@ public static RemoteStoreShardShallowCopySnapshot fromXContent(XContentParser pa
                     remoteStoreRepository = parser.text();
                 } else if (PARSE_REPOSITORY_BASE_PATH.match(currentFieldName, parser.getDeprecationHandler())) {
                     repositoryBasePath = parser.text();
+                } else if (PARSE_PATH_TYPE.match(currentFieldName, parser.getDeprecationHandler())) {
+                    pathType = PathType.fromCode(parser.intValue());
+                } else if (PARSE_PATH_HASH_ALGORITHM.match(currentFieldName, parser.getDeprecationHandler())) {
+                    pathHashAlgorithm = PathHashAlgorithm.fromCode(parser.intValue());
                 } else {
                     throw new OpenSearchParseException("unknown parameter [{}]", currentFieldName);
                 }
@@ -266,7 +293,9 @@ public static RemoteStoreShardShallowCopySnapshot fromXContent(XContentParser pa
             indexUUID,
             remoteStoreRepository,
             repositoryBasePath,
-            fileNames
+            fileNames,
+            pathType,
+            pathHashAlgorithm
         );
     }
 
@@ -380,38 +409,47 @@ private void verifyParameters(
         long commitGeneration,
         String indexUUID,
         String remoteStoreRepository,
-        String repositoryBasePath
+        String repositoryBasePath,
+        PathType pathType,
+        PathHashAlgorithm pathHashAlgorithm
     ) {
-        String exceptionStr = null;
-        if (version == null) {
-            exceptionStr = "Invalid Version Provided";
-        }
-        if (snapshot == null) {
-            exceptionStr = "Invalid/Missing Snapshot Name";
-        }
-        if (indexVersion < 0) {
-            exceptionStr = "Invalid Index Version";
-        }
-        if (primaryTerm < 0) {
-            exceptionStr = "Invalid Primary Term";
-        }
-        if (commitGeneration < 0) {
-            exceptionStr = "Invalid Commit Generation";
-        }
-        if (indexUUID == null) {
-            exceptionStr = "Invalid/Missing Index UUID";
-        }
-        if (remoteStoreRepository == null) {
-            exceptionStr = "Invalid/Missing Remote Store Repository";
-        }
-        if (repositoryBasePath == null) {
-            exceptionStr = "Invalid/Missing Repository Base Path";
-        }
-        if (exceptionStr != null) {
+
+        throwExceptionIfInvalid(Objects.isNull(version), "Invalid Version Provided");
+        throwExceptionIfInvalid(Objects.isNull(snapshot), "Invalid/Missing Snapshot Name");
+        throwExceptionIfInvalid(indexVersion < 0, "Invalid Index Version");
+        throwExceptionIfInvalid(primaryTerm < 0, "Invalid Primary Term");
+        throwExceptionIfInvalid(commitGeneration < 0, "Invalid Commit Generation");
+        throwExceptionIfInvalid(Objects.isNull(indexUUID), "Invalid/Missing Index UUID");
+        throwExceptionIfInvalid(Objects.isNull(remoteStoreRepository), "Invalid/Missing Remote Store Repository");
+        throwExceptionIfInvalid(Objects.isNull(repositoryBasePath), "Invalid/Missing Repository Base Path");
+        throwExceptionIfInvalid(
+            isValidRemotePathConfiguration(version, pathType, pathHashAlgorithm) == false,
+            new ParameterizedMessage(
+                "Invalid combination of pathType={} pathHashAlgorithm={} for version={}",
+                pathType,
+                pathHashAlgorithm,
+                version
+            ).getFormattedMessage()
+        );
+    }
+
+    private void throwExceptionIfInvalid(boolean isInvalid, String exceptionStr) {
+        if (isInvalid) {
             throw new IllegalArgumentException(exceptionStr);
         }
     }
 
+    private boolean isValidRemotePathConfiguration(String version, PathType pathType, PathHashAlgorithm pathHashAlgorithm) {
+        switch (version) {
+            case "1":
+                return Objects.isNull(pathType) && Objects.isNull(pathHashAlgorithm);
+            case "2":
+                return Objects.nonNull(pathType) && RemoteStorePathStrategy.isCompatible(pathType, pathHashAlgorithm);
+            default:
+                return false;
+        }
+    }
+
     /**
      * Creates a new instance which has a different name and zero incremental file counts but is identical to this instance in terms of the files
      * it references.
@@ -433,7 +471,9 @@ public RemoteStoreShardShallowCopySnapshot asClone(String targetSnapshotName, lo
             indexUUID,
             remoteStoreRepository,
             repositoryBasePath,
-            fileNames
+            fileNames,
+            pathType,
+            pathHashAlgorithm
         );
     }
 
@@ -449,4 +489,63 @@ public IndexShardSnapshotStatus getIndexShardSnapshotStatus() {
             null
         ); // Not adding a real generation here as it doesn't matter to callers
     }
+
+    public PathType getPathType() {
+        return pathType;
+    }
+
+    public PathHashAlgorithm getPathHashAlgorithm() {
+        return pathHashAlgorithm;
+    }
+
+    public RemoteStorePathStrategy getRemoteStorePathStrategy() {
+        if (Objects.nonNull(pathType)) {
+            return new RemoteStorePathStrategy(pathType, pathHashAlgorithm);
+        }
+        return new RemoteStorePathStrategy(PathType.FIXED);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) return true;
+        if (obj == null || getClass() != obj.getClass()) return false;
+        RemoteStoreShardShallowCopySnapshot that = (RemoteStoreShardShallowCopySnapshot) obj;
+
+        return Objects.equals(this.snapshot, that.snapshot)
+            && Objects.equals(this.version, that.version)
+            && this.indexVersion == that.indexVersion
+            && this.startTime == that.startTime
+            && this.time == that.time
+            && this.totalFileCount == that.totalFileCount
+            && this.totalSize == that.totalSize
+            && this.primaryTerm == that.primaryTerm
+            && this.commitGeneration == that.commitGeneration
+            && Objects.equals(this.remoteStoreRepository, that.remoteStoreRepository)
+            && Objects.equals(this.repositoryBasePath, that.repositoryBasePath)
+            && Objects.equals(this.indexUUID, that.indexUUID)
+            && Objects.equals(this.fileNames, that.fileNames)
+            && Objects.equals(this.pathType, that.pathType)
+            && Objects.equals(this.pathHashAlgorithm, that.pathHashAlgorithm);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(
+            snapshot,
+            version,
+            indexVersion,
+            startTime,
+            time,
+            totalFileCount,
+            totalSize,
+            primaryTerm,
+            commitGeneration,
+            remoteStoreRepository,
+            repositoryBasePath,
+            indexUUID,
+            fileNames,
+            pathType,
+            pathHashAlgorithm
+        );
+    }
 }
diff --git a/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java
index 345583bbbd1be..ab76150f8f83d 100644
--- a/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java
+++ b/server/src/main/java/org/opensearch/index/store/RemoteDirectory.java
@@ -29,6 +29,7 @@
 import org.opensearch.common.blobstore.transfer.stream.OffsetRangeIndexInputStream;
 import org.opensearch.common.blobstore.transfer.stream.OffsetRangeInputStream;
 import org.opensearch.core.action.ActionListener;
+import org.opensearch.core.common.unit.ByteSizeUnit;
 import org.opensearch.index.store.exception.ChecksumCombinationException;
 
 import java.io.FileNotFoundException;
@@ -323,11 +324,12 @@ public boolean copyFrom(
         String remoteFileName,
         IOContext context,
         Runnable postUploadRunner,
-        ActionListener<Void> listener
+        ActionListener<Void> listener,
+        boolean lowPriorityUpload
     ) {
         if (blobContainer instanceof AsyncMultiStreamBlobContainer) {
             try {
-                uploadBlob(from, src, remoteFileName, context, postUploadRunner, listener);
+                uploadBlob(from, src, remoteFileName, context, postUploadRunner, listener, lowPriorityUpload);
             } catch (Exception e) {
                 listener.onFailure(e);
             }
@@ -342,7 +344,8 @@ private void uploadBlob(
         String remoteFileName,
         IOContext ioContext,
         Runnable postUploadRunner,
-        ActionListener<Void> listener
+        ActionListener<Void> listener,
+        boolean lowPriorityUpload
     ) throws Exception {
         long expectedChecksum = calculateChecksumOfChecksum(from, src);
         long contentLength;
@@ -353,12 +356,13 @@ private void uploadBlob(
         if (getBlobContainer() instanceof AsyncMultiStreamBlobContainer) {
             remoteIntegrityEnabled = ((AsyncMultiStreamBlobContainer) getBlobContainer()).remoteIntegrityCheckSupported();
         }
+        lowPriorityUpload = lowPriorityUpload || contentLength > ByteSizeUnit.GB.toBytes(15);
         RemoteTransferContainer remoteTransferContainer = new RemoteTransferContainer(
             src,
             remoteFileName,
             contentLength,
             true,
-            WritePriority.NORMAL,
+            lowPriorityUpload ? WritePriority.LOW : WritePriority.NORMAL,
             (size, position) -> uploadRateLimiter.apply(new OffsetRangeIndexInputStream(from.openInput(src, ioContext), size, position)),
             expectedChecksum,
             remoteIntegrityEnabled
diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java
index c9a238c6e3350..8c0ecb4cc783a 100644
--- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java
+++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectory.java
@@ -31,6 +31,7 @@
 import org.opensearch.common.lucene.store.ByteArrayIndexInput;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.index.shard.ShardId;
+import org.opensearch.index.remote.RemoteStorePathStrategy;
 import org.opensearch.index.remote.RemoteStoreUtils;
 import org.opensearch.index.store.lockmanager.FileLockInfo;
 import org.opensearch.index.store.lockmanager.RemoteStoreCommitLevelLockManager;
@@ -452,7 +453,7 @@ public IndexInput openInput(String name, IOContext context) throws IOException {
      * @param context  IOContext to be used to open IndexInput of file during remote upload
      * @param listener Listener to handle upload callback events
      */
-    public void copyFrom(Directory from, String src, IOContext context, ActionListener<Void> listener) {
+    public void copyFrom(Directory from, String src, IOContext context, ActionListener<Void> listener, boolean lowPriorityUpload) {
         try {
             final String remoteFileName = getNewRemoteSegmentFilename(src);
             boolean uploaded = remoteDataDirectory.copyFrom(from, src, remoteFileName, context, () -> {
@@ -461,7 +462,7 @@ public void copyFrom(Directory from, String src, IOContext context, ActionListen
                 } catch (IOException e) {
                     throw new RuntimeException("Exception in segment postUpload for file " + src, e);
                 }
-            }, listener);
+            }, listener, lowPriorityUpload);
             if (uploaded == false) {
                 copyFrom(from, src, src, context);
                 listener.onResponse(null);
@@ -897,13 +898,15 @@ public static void remoteDirectoryCleanup(
         RemoteSegmentStoreDirectoryFactory remoteDirectoryFactory,
         String remoteStoreRepoForIndex,
         String indexUUID,
-        ShardId shardId
+        ShardId shardId,
+        RemoteStorePathStrategy pathStrategy
     ) {
         try {
             RemoteSegmentStoreDirectory remoteSegmentStoreDirectory = (RemoteSegmentStoreDirectory) remoteDirectoryFactory.newDirectory(
                 remoteStoreRepoForIndex,
                 indexUUID,
-                shardId
+                shardId,
+                pathStrategy
             );
             remoteSegmentStoreDirectory.deleteStaleSegments(0);
             remoteSegmentStoreDirectory.deleteIfEmpty();
diff --git a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java
index eca8d9ec702e1..e462f6d4ac011 100644
--- a/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java
+++ b/server/src/main/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryFactory.java
@@ -9,9 +9,11 @@
 package org.opensearch.index.store;
 
 import org.apache.lucene.store.Directory;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.blobstore.BlobPath;
 import org.opensearch.core.index.shard.ShardId;
 import org.opensearch.index.IndexSettings;
+import org.opensearch.index.remote.RemoteStorePathStrategy;
 import org.opensearch.index.shard.ShardPath;
 import org.opensearch.index.store.lockmanager.RemoteStoreLockManager;
 import org.opensearch.index.store.lockmanager.RemoteStoreLockManagerFactory;
@@ -23,16 +25,20 @@
 import org.opensearch.threadpool.ThreadPool;
 
 import java.io.IOException;
+import java.util.Objects;
 import java.util.function.Supplier;
 
+import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.SEGMENTS;
+import static org.opensearch.index.remote.RemoteStoreEnums.DataType.DATA;
+import static org.opensearch.index.remote.RemoteStoreEnums.DataType.METADATA;
+
 /**
  * Factory for a remote store directory
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.3.0")
 public class RemoteSegmentStoreDirectoryFactory implements IndexStorePlugin.DirectoryFactory {
-    private static final String SEGMENTS = "segments";
-
     private final Supplier<RepositoriesService> repositoriesService;
 
     private final ThreadPool threadPool;
@@ -46,29 +52,52 @@ public RemoteSegmentStoreDirectoryFactory(Supplier<RepositoriesService> reposito
     public Directory newDirectory(IndexSettings indexSettings, ShardPath path) throws IOException {
         String repositoryName = indexSettings.getRemoteStoreRepository();
         String indexUUID = indexSettings.getIndex().getUUID();
-        return newDirectory(repositoryName, indexUUID, path.getShardId());
+        return newDirectory(repositoryName, indexUUID, path.getShardId(), indexSettings.getRemoteStorePathStrategy());
     }
 
-    public Directory newDirectory(String repositoryName, String indexUUID, ShardId shardId) throws IOException {
+    public Directory newDirectory(String repositoryName, String indexUUID, ShardId shardId, RemoteStorePathStrategy pathStrategy)
+        throws IOException {
+        assert Objects.nonNull(pathStrategy);
         try (Repository repository = repositoriesService.get().repository(repositoryName)) {
+
             assert repository instanceof BlobStoreRepository : "repository should be instance of BlobStoreRepository";
             BlobStoreRepository blobStoreRepository = ((BlobStoreRepository) repository);
-            BlobPath commonBlobPath = blobStoreRepository.basePath();
-            commonBlobPath = commonBlobPath.add(indexUUID).add(String.valueOf(shardId.id())).add(SEGMENTS);
+            BlobPath repositoryBasePath = blobStoreRepository.basePath();
+            String shardIdStr = String.valueOf(shardId.id());
 
+            RemoteStorePathStrategy.PathInput dataPathInput = RemoteStorePathStrategy.PathInput.builder()
+                .basePath(repositoryBasePath)
+                .indexUUID(indexUUID)
+                .shardId(shardIdStr)
+                .dataCategory(SEGMENTS)
+                .dataType(DATA)
+                .build();
+            // Derive the path for data directory of SEGMENTS
+            BlobPath dataPath = pathStrategy.generatePath(dataPathInput);
             RemoteDirectory dataDirectory = new RemoteDirectory(
-                blobStoreRepository.blobStore().blobContainer(commonBlobPath.add("data")),
+                blobStoreRepository.blobStore().blobContainer(dataPath),
                 blobStoreRepository::maybeRateLimitRemoteUploadTransfers,
                 blobStoreRepository::maybeRateLimitRemoteDownloadTransfers
             );
-            RemoteDirectory metadataDirectory = new RemoteDirectory(
-                blobStoreRepository.blobStore().blobContainer(commonBlobPath.add("metadata"))
-            );
+
+            RemoteStorePathStrategy.PathInput mdPathInput = RemoteStorePathStrategy.PathInput.builder()
+                .basePath(repositoryBasePath)
+                .indexUUID(indexUUID)
+                .shardId(shardIdStr)
+                .dataCategory(SEGMENTS)
+                .dataType(METADATA)
+                .build();
+            // Derive the path for metadata directory of SEGMENTS
+            BlobPath mdPath = pathStrategy.generatePath(mdPathInput);
+            RemoteDirectory metadataDirectory = new RemoteDirectory(blobStoreRepository.blobStore().blobContainer(mdPath));
+
+            // The path for lock is derived within the RemoteStoreLockManagerFactory
             RemoteStoreLockManager mdLockManager = RemoteStoreLockManagerFactory.newLockManager(
                 repositoriesService.get(),
                 repositoryName,
                 indexUUID,
-                String.valueOf(shardId.id())
+                shardIdStr,
+                pathStrategy
             );
 
             return new RemoteSegmentStoreDirectory(dataDirectory, metadataDirectory, mdLockManager, threadPool, shardId);
diff --git a/server/src/main/java/org/opensearch/index/store/Store.java b/server/src/main/java/org/opensearch/index/store/Store.java
index 0992d86d6f0aa..56fc5b1ffa90d 100644
--- a/server/src/main/java/org/opensearch/index/store/Store.java
+++ b/server/src/main/java/org/opensearch/index/store/Store.java
@@ -893,7 +893,7 @@ public void beforeClose() {
      * @throws IOException when there is an IO error committing.
      */
     public void commitSegmentInfos(SegmentInfos latestSegmentInfos, long maxSeqNo, long processedCheckpoint) throws IOException {
-        assert indexSettings.isSegRepEnabledOrRemoteNode() || indexSettings.isRemoteNode();
+        assert indexSettings.isSegRepEnabledOrRemoteNode() || indexSettings.isAssignedOnRemoteNode();
         metadataLock.writeLock().lock();
         try {
             final Map<String, String> userData = new HashMap<>(latestSegmentInfos.getUserData());
diff --git a/server/src/main/java/org/opensearch/index/store/lockmanager/RemoteStoreLockManagerFactory.java b/server/src/main/java/org/opensearch/index/store/lockmanager/RemoteStoreLockManagerFactory.java
index 00666ada11983..45d466d3a8ce8 100644
--- a/server/src/main/java/org/opensearch/index/store/lockmanager/RemoteStoreLockManagerFactory.java
+++ b/server/src/main/java/org/opensearch/index/store/lockmanager/RemoteStoreLockManagerFactory.java
@@ -11,15 +11,18 @@
 import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.blobstore.BlobContainer;
 import org.opensearch.common.blobstore.BlobPath;
+import org.opensearch.index.remote.RemoteStorePathStrategy;
 import org.opensearch.index.store.RemoteBufferedOutputDirectory;
 import org.opensearch.repositories.RepositoriesService;
 import org.opensearch.repositories.Repository;
 import org.opensearch.repositories.RepositoryMissingException;
 import org.opensearch.repositories.blobstore.BlobStoreRepository;
 
-import java.io.IOException;
 import java.util.function.Supplier;
 
+import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.SEGMENTS;
+import static org.opensearch.index.remote.RemoteStoreEnums.DataType.LOCK_FILES;
+
 /**
  * Factory for remote store lock manager
  *
@@ -27,34 +30,42 @@
  */
 @PublicApi(since = "2.8.0")
 public class RemoteStoreLockManagerFactory {
-    private static final String SEGMENTS = "segments";
-    private static final String LOCK_FILES = "lock_files";
     private final Supplier<RepositoriesService> repositoriesService;
 
     public RemoteStoreLockManagerFactory(Supplier<RepositoriesService> repositoriesService) {
         this.repositoriesService = repositoriesService;
     }
 
-    public RemoteStoreLockManager newLockManager(String repositoryName, String indexUUID, String shardId) throws IOException {
-        return newLockManager(repositoriesService.get(), repositoryName, indexUUID, shardId);
+    public RemoteStoreLockManager newLockManager(
+        String repositoryName,
+        String indexUUID,
+        String shardId,
+        RemoteStorePathStrategy pathStrategy
+    ) {
+        return newLockManager(repositoriesService.get(), repositoryName, indexUUID, shardId, pathStrategy);
     }
 
     public static RemoteStoreMetadataLockManager newLockManager(
         RepositoriesService repositoriesService,
         String repositoryName,
         String indexUUID,
-        String shardId
-    ) throws IOException {
+        String shardId,
+        RemoteStorePathStrategy pathStrategy
+    ) {
         try (Repository repository = repositoriesService.repository(repositoryName)) {
             assert repository instanceof BlobStoreRepository : "repository should be instance of BlobStoreRepository";
-            BlobPath shardLevelBlobPath = ((BlobStoreRepository) repository).basePath().add(indexUUID).add(shardId).add(SEGMENTS);
-            RemoteBufferedOutputDirectory shardMDLockDirectory = createRemoteBufferedOutputDirectory(
-                repository,
-                shardLevelBlobPath,
-                LOCK_FILES
-            );
+            BlobPath repositoryBasePath = ((BlobStoreRepository) repository).basePath();
 
-            return new RemoteStoreMetadataLockManager(shardMDLockDirectory);
+            RemoteStorePathStrategy.PathInput lockFilesPathInput = RemoteStorePathStrategy.PathInput.builder()
+                .basePath(repositoryBasePath)
+                .indexUUID(indexUUID)
+                .shardId(shardId)
+                .dataCategory(SEGMENTS)
+                .dataType(LOCK_FILES)
+                .build();
+            BlobPath lockDirectoryPath = pathStrategy.generatePath(lockFilesPathInput);
+            BlobContainer lockDirectoryBlobContainer = ((BlobStoreRepository) repository).blobStore().blobContainer(lockDirectoryPath);
+            return new RemoteStoreMetadataLockManager(new RemoteBufferedOutputDirectory(lockDirectoryBlobContainer));
         } catch (RepositoryMissingException e) {
             throw new IllegalArgumentException("Repository should be present to acquire/release lock", e);
         }
@@ -65,14 +76,4 @@ public static RemoteStoreMetadataLockManager newLockManager(
     public Supplier<RepositoriesService> getRepositoriesService() {
         return repositoriesService;
     }
-
-    private static RemoteBufferedOutputDirectory createRemoteBufferedOutputDirectory(
-        Repository repository,
-        BlobPath commonBlobPath,
-        String extention
-    ) {
-        BlobPath extendedPath = commonBlobPath.add(extention);
-        BlobContainer dataBlobContainer = ((BlobStoreRepository) repository).blobStore().blobContainer(extendedPath);
-        return new RemoteBufferedOutputDirectory(dataBlobContainer);
-    }
 }
diff --git a/server/src/main/java/org/opensearch/index/translog/InternalTranslogManager.java b/server/src/main/java/org/opensearch/index/translog/InternalTranslogManager.java
index a22c538286a88..e2210217672ef 100644
--- a/server/src/main/java/org/opensearch/index/translog/InternalTranslogManager.java
+++ b/server/src/main/java/org/opensearch/index/translog/InternalTranslogManager.java
@@ -437,6 +437,12 @@ public String getTranslogUUID() {
      * @return if the translog should be flushed
      */
     public boolean shouldPeriodicallyFlush(long localCheckpointOfLastCommit, long flushThreshold) {
+        /*
+         * This can trigger flush depending upon translog's implementation
+         */
+        if (translog.shouldFlush()) {
+            return true;
+        }
         // This is the minimum seqNo that is referred in translog and considered for calculating translog size
         long minTranslogRefSeqNo = translog.getMinUnreferencedSeqNoInSegments(localCheckpointOfLastCommit + 1);
         final long minReferencedTranslogGeneration = translog.getMinGenerationForSeqNo(minTranslogRefSeqNo).translogFileGeneration;
diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteBlobStoreInternalTranslogFactory.java b/server/src/main/java/org/opensearch/index/translog/RemoteBlobStoreInternalTranslogFactory.java
index e100ffaabf13d..4599aa32325c1 100644
--- a/server/src/main/java/org/opensearch/index/translog/RemoteBlobStoreInternalTranslogFactory.java
+++ b/server/src/main/java/org/opensearch/index/translog/RemoteBlobStoreInternalTranslogFactory.java
@@ -9,6 +9,7 @@
 package org.opensearch.index.translog;
 
 import org.opensearch.index.remote.RemoteTranslogTransferTracker;
+import org.opensearch.indices.RemoteStoreSettings;
 import org.opensearch.repositories.RepositoriesService;
 import org.opensearch.repositories.Repository;
 import org.opensearch.repositories.RepositoryMissingException;
@@ -34,11 +35,14 @@ public class RemoteBlobStoreInternalTranslogFactory implements TranslogFactory {
 
     private final RemoteTranslogTransferTracker remoteTranslogTransferTracker;
 
+    private final RemoteStoreSettings remoteStoreSettings;
+
     public RemoteBlobStoreInternalTranslogFactory(
         Supplier<RepositoriesService> repositoriesServiceSupplier,
         ThreadPool threadPool,
         String repositoryName,
-        RemoteTranslogTransferTracker remoteTranslogTransferTracker
+        RemoteTranslogTransferTracker remoteTranslogTransferTracker,
+        RemoteStoreSettings remoteStoreSettings
     ) {
         Repository repository;
         try {
@@ -49,6 +53,7 @@ public RemoteBlobStoreInternalTranslogFactory(
         this.repository = repository;
         this.threadPool = threadPool;
         this.remoteTranslogTransferTracker = remoteTranslogTransferTracker;
+        this.remoteStoreSettings = remoteStoreSettings;
     }
 
     @Override
@@ -74,7 +79,8 @@ public Translog newTranslog(
             blobStoreRepository,
             threadPool,
             startedPrimarySupplier,
-            remoteTranslogTransferTracker
+            remoteTranslogTransferTracker,
+            remoteStoreSettings
         );
     }
 
diff --git a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java
index 43eec01b2d365..c7f756957076c 100644
--- a/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java
+++ b/server/src/main/java/org/opensearch/index/translog/RemoteFsTranslog.java
@@ -11,6 +11,7 @@
 import org.apache.logging.log4j.Logger;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.SetOnce;
+import org.opensearch.common.blobstore.BlobPath;
 import org.opensearch.common.lease.Releasable;
 import org.opensearch.common.lease.Releasables;
 import org.opensearch.common.logging.Loggers;
@@ -18,7 +19,9 @@
 import org.opensearch.common.util.io.IOUtils;
 import org.opensearch.core.index.shard.ShardId;
 import org.opensearch.core.util.FileSystemUtils;
+import org.opensearch.index.remote.RemoteStorePathStrategy;
 import org.opensearch.index.remote.RemoteTranslogTransferTracker;
+import org.opensearch.index.seqno.SequenceNumbers;
 import org.opensearch.index.translog.transfer.BlobStoreTransferService;
 import org.opensearch.index.translog.transfer.FileTransferTracker;
 import org.opensearch.index.translog.transfer.TransferSnapshot;
@@ -26,6 +29,7 @@
 import org.opensearch.index.translog.transfer.TranslogTransferManager;
 import org.opensearch.index.translog.transfer.TranslogTransferMetadata;
 import org.opensearch.index.translog.transfer.listener.TranslogTransferListener;
+import org.opensearch.indices.RemoteStoreSettings;
 import org.opensearch.repositories.Repository;
 import org.opensearch.repositories.blobstore.BlobStoreRepository;
 import org.opensearch.threadpool.ThreadPool;
@@ -38,6 +42,7 @@
 import java.util.HashSet;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.Semaphore;
 import java.util.concurrent.TimeUnit;
@@ -47,6 +52,10 @@
 import java.util.function.LongConsumer;
 import java.util.function.LongSupplier;
 
+import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.TRANSLOG;
+import static org.opensearch.index.remote.RemoteStoreEnums.DataType.DATA;
+import static org.opensearch.index.remote.RemoteStoreEnums.DataType.METADATA;
+
 /**
  * A Translog implementation which syncs local FS with a remote store
  * The current impl uploads translog , ckp and metadata to remote store
@@ -74,7 +83,6 @@ public class RemoteFsTranslog extends Translog {
 
     private static final int REMOTE_DELETION_PERMITS = 2;
     private static final int DOWNLOAD_RETRIES = 2;
-    public static final String TRANSLOG = "translog";
 
     // Semaphore used to allow only single remote generation to happen at a time
     private final Semaphore remoteGenerationDeletionPermits = new Semaphore(REMOTE_DELETION_PERMITS);
@@ -94,7 +102,8 @@ public RemoteFsTranslog(
         BlobStoreRepository blobStoreRepository,
         ThreadPool threadPool,
         BooleanSupplier startedPrimarySupplier,
-        RemoteTranslogTransferTracker remoteTranslogTransferTracker
+        RemoteTranslogTransferTracker remoteTranslogTransferTracker,
+        RemoteStoreSettings remoteStoreSettings
     ) throws IOException {
         super(config, translogUUID, deletionPolicy, globalCheckpointSupplier, primaryTermSupplier, persistedSequenceNumberConsumer);
         logger = Loggers.getLogger(getClass(), shardId);
@@ -106,10 +115,12 @@ public RemoteFsTranslog(
             threadPool,
             shardId,
             fileTransferTracker,
-            remoteTranslogTransferTracker
+            remoteTranslogTransferTracker,
+            indexSettings().getRemoteStorePathStrategy(),
+            remoteStoreSettings
         );
         try {
-            download(translogTransferManager, location, logger);
+            download(translogTransferManager, location, logger, config.shouldSeedRemote());
             Checkpoint checkpoint = readCheckpoint(location);
             logger.info("Downloaded data from remote translog till maxSeqNo = {}", checkpoint.maxSeqNo);
             this.readers.addAll(recoverFromFiles(checkpoint));
@@ -150,8 +161,16 @@ RemoteTranslogTransferTracker getRemoteTranslogTracker() {
         return remoteTranslogTransferTracker;
     }
 
-    public static void download(Repository repository, ShardId shardId, ThreadPool threadPool, Path location, Logger logger)
-        throws IOException {
+    public static void download(
+        Repository repository,
+        ShardId shardId,
+        ThreadPool threadPool,
+        Path location,
+        RemoteStorePathStrategy pathStrategy,
+        RemoteStoreSettings remoteStoreSettings,
+        Logger logger,
+        boolean seedRemote
+    ) throws IOException {
         assert repository instanceof BlobStoreRepository : String.format(
             Locale.ROOT,
             "%s repository should be instance of BlobStoreRepository",
@@ -167,13 +186,16 @@ public static void download(Repository repository, ShardId shardId, ThreadPool t
             threadPool,
             shardId,
             fileTransferTracker,
-            remoteTranslogTransferTracker
+            remoteTranslogTransferTracker,
+            pathStrategy,
+            remoteStoreSettings
         );
-        RemoteFsTranslog.download(translogTransferManager, location, logger);
+        RemoteFsTranslog.download(translogTransferManager, location, logger, seedRemote);
         logger.trace(remoteTranslogTransferTracker.toString());
     }
 
-    static void download(TranslogTransferManager translogTransferManager, Path location, Logger logger) throws IOException {
+    static void download(TranslogTransferManager translogTransferManager, Path location, Logger logger, boolean seedRemote)
+        throws IOException {
         /*
         In Primary to Primary relocation , there can be concurrent upload and download of translog.
         While translog files are getting downloaded by new primary, it might hence be deleted by the primary
@@ -186,7 +208,7 @@ static void download(TranslogTransferManager translogTransferManager, Path locat
             boolean success = false;
             long startTimeMs = System.currentTimeMillis();
             try {
-                downloadOnce(translogTransferManager, location, logger);
+                downloadOnce(translogTransferManager, location, logger, seedRemote);
                 success = true;
                 return;
             } catch (FileNotFoundException | NoSuchFileException e) {
@@ -200,7 +222,8 @@ static void download(TranslogTransferManager translogTransferManager, Path locat
         throw ex;
     }
 
-    static private void downloadOnce(TranslogTransferManager translogTransferManager, Path location, Logger logger) throws IOException {
+    private static void downloadOnce(TranslogTransferManager translogTransferManager, Path location, Logger logger, boolean seedRemote)
+        throws IOException {
         logger.debug("Downloading translog files from remote");
         RemoteTranslogTransferTracker statsTracker = translogTransferManager.getRemoteTranslogTransferTracker();
         long prevDownloadBytesSucceeded = statsTracker.getDownloadBytesSucceeded();
@@ -235,24 +258,64 @@ static private void downloadOnce(TranslogTransferManager translogTransferManager
                 location.resolve(Translog.getCommitCheckpointFileName(translogMetadata.getGeneration())),
                 location.resolve(Translog.CHECKPOINT_FILE_NAME)
             );
+        } else {
+            // When code flow reaches this block, it means we don't have any translog files uploaded to remote store.
+            // If local filesystem contains empty translog or no translog, we don't do anything.
+            // If local filesystem contains non-empty translog, we clean up these files and create empty translog.
+            logger.debug("No translog files found on remote, checking local filesystem for cleanup");
+            if (FileSystemUtils.exists(location.resolve(CHECKPOINT_FILE_NAME))) {
+                final Checkpoint checkpoint = readCheckpoint(location);
+                if (seedRemote) {
+                    logger.debug("Remote migration ongoing. Retaining the translog on local, skipping clean-up");
+                } else if (isEmptyTranslog(checkpoint) == false) {
+                    logger.debug("Translog files exist on local without any metadata in remote, cleaning up these files");
+                    // Creating empty translog will cleanup the older un-referenced tranlog files, we don't have to explicitly delete
+                    Translog.createEmptyTranslog(location, translogTransferManager.getShardId(), checkpoint);
+                } else {
+                    logger.debug("Empty translog on local, skipping clean-up");
+                }
+            }
         }
         logger.debug("downloadOnce execution completed");
     }
 
+    private static boolean isEmptyTranslog(Checkpoint checkpoint) {
+        return checkpoint.generation == checkpoint.minTranslogGeneration
+            && checkpoint.minSeqNo == SequenceNumbers.NO_OPS_PERFORMED
+            && checkpoint.maxSeqNo == SequenceNumbers.NO_OPS_PERFORMED
+            && checkpoint.numOps == 0;
+    }
+
     public static TranslogTransferManager buildTranslogTransferManager(
         BlobStoreRepository blobStoreRepository,
         ThreadPool threadPool,
         ShardId shardId,
         FileTransferTracker fileTransferTracker,
-        RemoteTranslogTransferTracker remoteTranslogTransferTracker
+        RemoteTranslogTransferTracker tracker,
+        RemoteStorePathStrategy pathStrategy,
+        RemoteStoreSettings remoteStoreSettings
     ) {
-        return new TranslogTransferManager(
-            shardId,
-            new BlobStoreTransferService(blobStoreRepository.blobStore(), threadPool),
-            blobStoreRepository.basePath().add(shardId.getIndex().getUUID()).add(String.valueOf(shardId.id())).add(TRANSLOG),
-            fileTransferTracker,
-            remoteTranslogTransferTracker
-        );
+        assert Objects.nonNull(pathStrategy);
+        String indexUUID = shardId.getIndex().getUUID();
+        String shardIdStr = String.valueOf(shardId.id());
+        RemoteStorePathStrategy.PathInput dataPathInput = RemoteStorePathStrategy.PathInput.builder()
+            .basePath(blobStoreRepository.basePath())
+            .indexUUID(indexUUID)
+            .shardId(shardIdStr)
+            .dataCategory(TRANSLOG)
+            .dataType(DATA)
+            .build();
+        BlobPath dataPath = pathStrategy.generatePath(dataPathInput);
+        RemoteStorePathStrategy.PathInput mdPathInput = RemoteStorePathStrategy.PathInput.builder()
+            .basePath(blobStoreRepository.basePath())
+            .indexUUID(indexUUID)
+            .shardId(shardIdStr)
+            .dataCategory(TRANSLOG)
+            .dataType(METADATA)
+            .build();
+        BlobPath mdPath = pathStrategy.generatePath(mdPathInput);
+        BlobStoreTransferService transferService = new BlobStoreTransferService(blobStoreRepository.blobStore(), threadPool);
+        return new TranslogTransferManager(shardId, transferService, dataPath, mdPath, fileTransferTracker, tracker, remoteStoreSettings);
     }
 
     @Override
@@ -524,7 +587,13 @@ private void deleteStaleRemotePrimaryTerms() {
         }
     }
 
-    public static void cleanup(Repository repository, ShardId shardId, ThreadPool threadPool) throws IOException {
+    public static void cleanup(
+        Repository repository,
+        ShardId shardId,
+        ThreadPool threadPool,
+        RemoteStorePathStrategy pathStrategy,
+        RemoteStoreSettings remoteStoreSettings
+    ) throws IOException {
         assert repository instanceof BlobStoreRepository : "repository should be instance of BlobStoreRepository";
         BlobStoreRepository blobStoreRepository = (BlobStoreRepository) repository;
         // We use a dummy stats tracker to ensure the flow doesn't break.
@@ -536,7 +605,9 @@ public static void cleanup(Repository repository, ShardId shardId, ThreadPool th
             threadPool,
             shardId,
             fileTransferTracker,
-            remoteTranslogTransferTracker
+            remoteTranslogTransferTracker,
+            pathStrategy,
+            remoteStoreSettings
         );
         // clean up all remote translog files
         translogTransferManager.deleteTranslogFiles();
@@ -609,4 +680,15 @@ public long getMinUnreferencedSeqNoInSegments(long minUnrefCheckpointInLastCommi
     int availablePermits() {
         return syncPermit.availablePermits();
     }
+
+    /**
+     * Checks whether or not the shard should be flushed based on translog files.
+     * This checks if number of translog files breaches the threshold count determined by
+     * {@code cluster.remote_store.translog.max_readers} setting
+     * @return {@code true} if the shard should be flushed
+     */
+    @Override
+    protected boolean shouldFlush() {
+        return readers.size() >= translogTransferManager.getMaxRemoteTranslogReadersSettings();
+    }
 }
diff --git a/server/src/main/java/org/opensearch/index/translog/Translog.java b/server/src/main/java/org/opensearch/index/translog/Translog.java
index e78300e368099..842e9c77d2350 100644
--- a/server/src/main/java/org/opensearch/index/translog/Translog.java
+++ b/server/src/main/java/org/opensearch/index/translog/Translog.java
@@ -525,7 +525,7 @@ TranslogWriter createWriter(
                 tragedy,
                 persistedSequenceNumberConsumer,
                 bigArrays,
-                indexSettings.isRemoteNode()
+                indexSettings.isAssignedOnRemoteNode()
             );
         } catch (final IOException e) {
             throw new TranslogException(shardId, "failed to create new translog file", e);
@@ -2011,17 +2011,47 @@ public static String createEmptyTranslog(
         final long primaryTerm,
         @Nullable final String translogUUID,
         @Nullable final ChannelFactory factory
+    ) throws IOException {
+        return createEmptyTranslog(location, shardId, initialGlobalCheckpoint, primaryTerm, translogUUID, factory, 1);
+    }
+
+    public static String createEmptyTranslog(final Path location, final ShardId shardId, Checkpoint checkpoint) throws IOException {
+        final Path highestGenTranslogFile = location.resolve(getFilename(checkpoint.generation));
+        final TranslogHeader translogHeader;
+        try (FileChannel channel = FileChannel.open(highestGenTranslogFile, StandardOpenOption.READ)) {
+            translogHeader = TranslogHeader.read(highestGenTranslogFile, channel);
+        }
+        final String translogUUID = translogHeader.getTranslogUUID();
+        final long primaryTerm = translogHeader.getPrimaryTerm();
+        final ChannelFactory channelFactory = FileChannel::open;
+        return Translog.createEmptyTranslog(
+            location,
+            shardId,
+            SequenceNumbers.NO_OPS_PERFORMED,
+            primaryTerm,
+            translogUUID,
+            channelFactory,
+            checkpoint.generation + 1
+        );
+    }
+
+    public static String createEmptyTranslog(
+        final Path location,
+        final ShardId shardId,
+        final long initialGlobalCheckpoint,
+        final long primaryTerm,
+        @Nullable final String translogUUID,
+        @Nullable final ChannelFactory factory,
+        final long generation
     ) throws IOException {
         IOUtils.rm(location);
         Files.createDirectories(location);
 
-        final long generation = 1L;
-        final long minTranslogGeneration = 1L;
         final ChannelFactory channelFactory = factory != null ? factory : FileChannel::open;
         final String uuid = Strings.hasLength(translogUUID) ? translogUUID : UUIDs.randomBase64UUID();
         final Path checkpointFile = location.resolve(CHECKPOINT_FILE_NAME);
         final Path translogFile = location.resolve(getFilename(generation));
-        final Checkpoint checkpoint = Checkpoint.emptyTranslogCheckpoint(0, generation, initialGlobalCheckpoint, minTranslogGeneration);
+        final Checkpoint checkpoint = Checkpoint.emptyTranslogCheckpoint(0, generation, initialGlobalCheckpoint, generation);
 
         Checkpoint.write(channelFactory, checkpointFile, checkpoint, StandardOpenOption.WRITE, StandardOpenOption.CREATE_NEW);
         final TranslogWriter writer = TranslogWriter.create(
@@ -2031,7 +2061,7 @@ public static String createEmptyTranslog(
             translogFile,
             channelFactory,
             EMPTY_TRANSLOG_BUFFER_SIZE,
-            minTranslogGeneration,
+            generation,
             initialGlobalCheckpoint,
             () -> {
                 throw new UnsupportedOperationException();
@@ -2052,4 +2082,13 @@ public static String createEmptyTranslog(
     public long getMinUnreferencedSeqNoInSegments(long minUnrefCheckpointInLastCommit) {
         return minUnrefCheckpointInLastCommit;
     }
+
+    /**
+     * Checks whether or not the shard should be flushed based on translog files.
+     * each translog type can have it's own decider
+     * @return {@code true} if the shard should be flushed
+     */
+    protected boolean shouldFlush() {
+        return false;
+    }
 }
diff --git a/server/src/main/java/org/opensearch/index/translog/TranslogConfig.java b/server/src/main/java/org/opensearch/index/translog/TranslogConfig.java
index 2f00773075d41..f720f041b287c 100644
--- a/server/src/main/java/org/opensearch/index/translog/TranslogConfig.java
+++ b/server/src/main/java/org/opensearch/index/translog/TranslogConfig.java
@@ -59,6 +59,7 @@ public final class TranslogConfig {
     private final Path translogPath;
     private final ByteSizeValue bufferSize;
     private final String nodeId;
+    private final boolean seedRemote;
 
     /**
      * Creates a new TranslogConfig instance
@@ -66,9 +67,17 @@ public final class TranslogConfig {
      * @param translogPath the path to use for the transaction log files
      * @param indexSettings the index settings used to set internal variables
      * @param bigArrays a bigArrays instance used for temporarily allocating write operations
+     * @param seedRemote boolean denoting whether remote store needs to be seeded as part of remote migration
      */
-    public TranslogConfig(ShardId shardId, Path translogPath, IndexSettings indexSettings, BigArrays bigArrays, String nodeId) {
-        this(shardId, translogPath, indexSettings, bigArrays, DEFAULT_BUFFER_SIZE, nodeId);
+    public TranslogConfig(
+        ShardId shardId,
+        Path translogPath,
+        IndexSettings indexSettings,
+        BigArrays bigArrays,
+        String nodeId,
+        boolean seedRemote
+    ) {
+        this(shardId, translogPath, indexSettings, bigArrays, DEFAULT_BUFFER_SIZE, nodeId, seedRemote);
     }
 
     TranslogConfig(
@@ -77,7 +86,8 @@ public TranslogConfig(ShardId shardId, Path translogPath, IndexSettings indexSet
         IndexSettings indexSettings,
         BigArrays bigArrays,
         ByteSizeValue bufferSize,
-        String nodeId
+        String nodeId,
+        boolean seedRemote
     ) {
         this.bufferSize = bufferSize;
         this.indexSettings = indexSettings;
@@ -85,6 +95,7 @@ public TranslogConfig(ShardId shardId, Path translogPath, IndexSettings indexSet
         this.translogPath = translogPath;
         this.bigArrays = bigArrays;
         this.nodeId = nodeId;
+        this.seedRemote = seedRemote;
     }
 
     /**
@@ -125,4 +136,8 @@ public ByteSizeValue getBufferSize() {
     public String getNodeId() {
         return nodeId;
     }
+
+    public boolean shouldSeedRemote() {
+        return seedRemote;
+    }
 }
diff --git a/server/src/main/java/org/opensearch/index/translog/TruncateTranslogAction.java b/server/src/main/java/org/opensearch/index/translog/TruncateTranslogAction.java
index 25fcdc614172a..cf6c9087777fe 100644
--- a/server/src/main/java/org/opensearch/index/translog/TruncateTranslogAction.java
+++ b/server/src/main/java/org/opensearch/index/translog/TruncateTranslogAction.java
@@ -195,7 +195,8 @@ private boolean isTranslogClean(ShardPath shardPath, ClusterState clusterState,
                 translogPath,
                 indexSettings,
                 BigArrays.NON_RECYCLING_INSTANCE,
-                ""
+                "",
+                false
             );
             long primaryTerm = indexSettings.getIndexMetadata().primaryTerm(shardPath.getShardId().id());
             // We open translog to check for corruption, do not clean anything.
diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/BlobStoreTransferService.java b/server/src/main/java/org/opensearch/index/translog/transfer/BlobStoreTransferService.java
index 82dd6301ef79f..bec2d78d9af62 100644
--- a/server/src/main/java/org/opensearch/index/translog/transfer/BlobStoreTransferService.java
+++ b/server/src/main/java/org/opensearch/index/translog/transfer/BlobStoreTransferService.java
@@ -12,11 +12,13 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.opensearch.action.ActionRunnable;
+import org.opensearch.common.annotation.ExperimentalApi;
 import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer;
 import org.opensearch.common.blobstore.BlobContainer;
 import org.opensearch.common.blobstore.BlobMetadata;
 import org.opensearch.common.blobstore.BlobPath;
 import org.opensearch.common.blobstore.BlobStore;
+import org.opensearch.common.blobstore.FetchBlobResult;
 import org.opensearch.common.blobstore.stream.write.WriteContext;
 import org.opensearch.common.blobstore.stream.write.WritePriority;
 import org.opensearch.common.blobstore.transfer.RemoteTransferContainer;
@@ -164,6 +166,12 @@ public InputStream downloadBlob(Iterable<String> path, String fileName) throws I
         return blobStore.blobContainer((BlobPath) path).readBlob(fileName);
     }
 
+    @Override
+    @ExperimentalApi
+    public FetchBlobResult downloadBlobWithMetadata(Iterable<String> path, String fileName) throws IOException {
+        return blobStore.blobContainer((BlobPath) path).readBlobWithMetadata(fileName);
+    }
+
     @Override
     public void deleteBlobs(Iterable<String> path, List<String> fileNames) throws IOException {
         blobStore.blobContainer((BlobPath) path).deleteBlobsIgnoringIfNotExists(fileNames);
diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/TransferService.java b/server/src/main/java/org/opensearch/index/translog/transfer/TransferService.java
index cfe833dde87eb..0894ebf500ebd 100644
--- a/server/src/main/java/org/opensearch/index/translog/transfer/TransferService.java
+++ b/server/src/main/java/org/opensearch/index/translog/transfer/TransferService.java
@@ -8,8 +8,10 @@
 
 package org.opensearch.index.translog.transfer;
 
+import org.opensearch.common.annotation.ExperimentalApi;
 import org.opensearch.common.blobstore.BlobMetadata;
 import org.opensearch.common.blobstore.BlobPath;
+import org.opensearch.common.blobstore.FetchBlobResult;
 import org.opensearch.common.blobstore.stream.write.WritePriority;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.index.translog.transfer.FileSnapshot.TransferFileSnapshot;
@@ -125,6 +127,16 @@ void uploadBlobs(
      */
     InputStream downloadBlob(Iterable<String> path, String fileName) throws IOException;
 
+    /**
+     *
+     * @param path  the remote path from where download should be made
+     * @param fileName the name of the file
+     * @return {@link FetchBlobResult} of the remote file
+     * @throws IOException the exception while reading the data
+     */
+    @ExperimentalApi
+    FetchBlobResult downloadBlobWithMetadata(Iterable<String> path, String fileName) throws IOException;
+
     void listAllInSortedOrder(Iterable<String> path, String filenamePrefix, int limit, ActionListener<List<BlobMetadata>> listener);
 
     void listAllInSortedOrderAsync(
diff --git a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java
index 2f6055df87804..47638f44fd6fc 100644
--- a/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java
+++ b/server/src/main/java/org/opensearch/index/translog/transfer/TranslogTransferManager.java
@@ -28,6 +28,7 @@
 import org.opensearch.index.remote.RemoteTranslogTransferTracker;
 import org.opensearch.index.translog.Translog;
 import org.opensearch.index.translog.transfer.listener.TranslogTransferListener;
+import org.opensearch.indices.RemoteStoreSettings;
 import org.opensearch.threadpool.ThreadPool;
 
 import java.io.IOException;
@@ -58,17 +59,12 @@ public class TranslogTransferManager {
     private final TransferService transferService;
     private final BlobPath remoteDataTransferPath;
     private final BlobPath remoteMetadataTransferPath;
-    private final BlobPath remoteBaseTransferPath;
     private final FileTransferTracker fileTransferTracker;
     private final RemoteTranslogTransferTracker remoteTranslogTransferTracker;
-
-    private static final long TRANSFER_TIMEOUT_IN_MILLIS = 30000;
-
+    private final RemoteStoreSettings remoteStoreSettings;
     private static final int METADATA_FILES_TO_FETCH = 10;
 
     private final Logger logger;
-    private final static String METADATA_DIR = "metadata";
-    private final static String DATA_DIR = "data";
 
     private static final VersionedCodecStreamWrapper<TranslogTransferMetadata> metadataStreamWrapper = new VersionedCodecStreamWrapper<>(
         new TranslogTransferMetadataHandler(),
@@ -79,18 +75,20 @@ public class TranslogTransferManager {
     public TranslogTransferManager(
         ShardId shardId,
         TransferService transferService,
-        BlobPath remoteBaseTransferPath,
+        BlobPath remoteDataTransferPath,
+        BlobPath remoteMetadataTransferPath,
         FileTransferTracker fileTransferTracker,
-        RemoteTranslogTransferTracker remoteTranslogTransferTracker
+        RemoteTranslogTransferTracker remoteTranslogTransferTracker,
+        RemoteStoreSettings remoteStoreSettings
     ) {
         this.shardId = shardId;
         this.transferService = transferService;
-        this.remoteBaseTransferPath = remoteBaseTransferPath;
-        this.remoteDataTransferPath = remoteBaseTransferPath.add(DATA_DIR);
-        this.remoteMetadataTransferPath = remoteBaseTransferPath.add(METADATA_DIR);
+        this.remoteDataTransferPath = remoteDataTransferPath;
+        this.remoteMetadataTransferPath = remoteMetadataTransferPath;
         this.fileTransferTracker = fileTransferTracker;
         this.logger = Loggers.getLogger(getClass(), shardId);
         this.remoteTranslogTransferTracker = remoteTranslogTransferTracker;
+        this.remoteStoreSettings = remoteStoreSettings;
     }
 
     public RemoteTranslogTransferTracker getRemoteTranslogTransferTracker() {
@@ -154,7 +152,7 @@ public boolean transferSnapshot(TransferSnapshot transferSnapshot, TranslogTrans
             transferService.uploadBlobs(toUpload, blobPathMap, latchedActionListener, WritePriority.HIGH);
 
             try {
-                if (latch.await(TRANSFER_TIMEOUT_IN_MILLIS, TimeUnit.MILLISECONDS) == false) {
+                if (latch.await(remoteStoreSettings.getClusterRemoteTranslogTransferTimeout().millis(), TimeUnit.MILLISECONDS) == false) {
                     Exception ex = new TranslogUploadFailedException(
                         "Timed out waiting for transfer of snapshot " + transferSnapshot + " to complete"
                     );
@@ -456,17 +454,27 @@ public void onFailure(Exception e) {
         );
     }
 
+    /**
+     * Deletes all the translog content related to the underlying shard.
+     */
     public void delete() {
-        // cleans up all the translog contents in async fashion
-        transferService.deleteAsync(ThreadPool.Names.REMOTE_PURGE, remoteBaseTransferPath, new ActionListener<>() {
+        // Delete the translog data content from the remote store.
+        delete(remoteDataTransferPath);
+        // Delete the translog metadata content from the remote store.
+        delete(remoteMetadataTransferPath);
+    }
+
+    private void delete(BlobPath path) {
+        // cleans up all the translog contents in async fashion for the given path
+        transferService.deleteAsync(ThreadPool.Names.REMOTE_PURGE, path, new ActionListener<>() {
             @Override
             public void onResponse(Void unused) {
-                logger.info("Deleted all remote translog data");
+                logger.info("Deleted all remote translog data at path={}", path);
             }
 
             @Override
             public void onFailure(Exception e) {
-                logger.error("Exception occurred while cleaning translog", e);
+                logger.error(new ParameterizedMessage("Exception occurred while cleaning translog at path={}", path), e);
             }
         });
     }
@@ -577,4 +585,8 @@ public void onFailure(Exception e) {
             throw e;
         }
     }
+
+    public int getMaxRemoteTranslogReadersSettings() {
+        return this.remoteStoreSettings.getMaxRemoteTranslogReaders();
+    }
 }
diff --git a/server/src/main/java/org/opensearch/indices/AbstractIndexShardCacheEntity.java b/server/src/main/java/org/opensearch/indices/AbstractIndexShardCacheEntity.java
index bb1201cb910a9..6b4c53654d871 100644
--- a/server/src/main/java/org/opensearch/indices/AbstractIndexShardCacheEntity.java
+++ b/server/src/main/java/org/opensearch/indices/AbstractIndexShardCacheEntity.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.indices;
 
+import org.opensearch.common.cache.ICacheKey;
 import org.opensearch.common.cache.RemovalNotification;
 import org.opensearch.common.cache.RemovalReason;
 import org.opensearch.core.common.bytes.BytesReference;
@@ -51,8 +52,8 @@ abstract class AbstractIndexShardCacheEntity implements IndicesRequestCache.Cach
     protected abstract ShardRequestCache stats();
 
     @Override
-    public final void onCached(IndicesRequestCache.Key key, BytesReference value) {
-        stats().onCached(key, value);
+    public final void onCached(ICacheKey<IndicesRequestCache.Key> key, BytesReference value) {
+        stats().onCached(getRamBytesUsedInKey(key), value);
     }
 
     @Override
@@ -66,7 +67,19 @@ public final void onMiss() {
     }
 
     @Override
-    public final void onRemoval(RemovalNotification<IndicesRequestCache.Key, BytesReference> notification) {
-        stats().onRemoval(notification.getKey(), notification.getValue(), notification.getRemovalReason() == RemovalReason.EVICTED);
+    public final void onRemoval(RemovalNotification<ICacheKey<IndicesRequestCache.Key>, BytesReference> notification) {
+        stats().onRemoval(
+            getRamBytesUsedInKey(notification.getKey()),
+            notification.getValue(),
+            notification.getRemovalReason() == RemovalReason.EVICTED
+        );
+    }
+
+    private long getRamBytesUsedInKey(ICacheKey<IndicesRequestCache.Key> key) {
+        long innerKeyRamBytesUsed = 0;
+        if (key.key != null) {
+            innerKeyRamBytesUsed = key.key.ramBytesUsed();
+        }
+        return key.ramBytesUsed(innerKeyRamBytesUsed);
     }
 }
diff --git a/server/src/main/java/org/opensearch/indices/DefaultRemoteStoreSettings.java b/server/src/main/java/org/opensearch/indices/DefaultRemoteStoreSettings.java
new file mode 100644
index 0000000000000..d3937600a848b
--- /dev/null
+++ b/server/src/main/java/org/opensearch/indices/DefaultRemoteStoreSettings.java
@@ -0,0 +1,26 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.indices;
+
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Settings;
+
+/**
+ * Utility to provide a {@link RemoteStoreSettings} instance containing all defaults
+ *
+ * @opensearch.internal
+ */
+public final class DefaultRemoteStoreSettings {
+    private DefaultRemoteStoreSettings() {}
+
+    public static final RemoteStoreSettings INSTANCE = new RemoteStoreSettings(
+        Settings.EMPTY,
+        new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
+    );
+}
diff --git a/server/src/main/java/org/opensearch/indices/IndicesModule.java b/server/src/main/java/org/opensearch/indices/IndicesModule.java
index b86e98f4ebcbc..d2c26157b1963 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesModule.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesModule.java
@@ -46,8 +46,10 @@
 import org.opensearch.index.mapper.BinaryFieldMapper;
 import org.opensearch.index.mapper.BooleanFieldMapper;
 import org.opensearch.index.mapper.CompletionFieldMapper;
+import org.opensearch.index.mapper.ConstantKeywordFieldMapper;
 import org.opensearch.index.mapper.DataStreamFieldMapper;
 import org.opensearch.index.mapper.DateFieldMapper;
+import org.opensearch.index.mapper.DerivedFieldMapper;
 import org.opensearch.index.mapper.DocCountFieldMapper;
 import org.opensearch.index.mapper.FieldAliasMapper;
 import org.opensearch.index.mapper.FieldNamesFieldMapper;
@@ -168,6 +170,8 @@ public static Map<String, Mapper.TypeParser> getMappers(List<MapperPlugin> mappe
         mappers.put(FieldAliasMapper.CONTENT_TYPE, new FieldAliasMapper.TypeParser());
         mappers.put(GeoPointFieldMapper.CONTENT_TYPE, new GeoPointFieldMapper.TypeParser());
         mappers.put(FlatObjectFieldMapper.CONTENT_TYPE, FlatObjectFieldMapper.PARSER);
+        mappers.put(ConstantKeywordFieldMapper.CONTENT_TYPE, new ConstantKeywordFieldMapper.TypeParser());
+        mappers.put(DerivedFieldMapper.CONTENT_TYPE, DerivedFieldMapper.PARSER);
 
         for (MapperPlugin mapperPlugin : mapperPlugins) {
             for (Map.Entry<String, Mapper.TypeParser> entry : mapperPlugin.getMappers().entrySet()) {
diff --git a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
index 34c8d6cf5e840..44af83bb839c1 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesRequestCache.java
@@ -39,16 +39,21 @@
 import org.apache.lucene.util.Accountable;
 import org.apache.lucene.util.RamUsageEstimator;
 import org.opensearch.OpenSearchParseException;
+import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.CheckedSupplier;
 import org.opensearch.common.cache.CacheType;
 import org.opensearch.common.cache.ICache;
+import org.opensearch.common.cache.ICacheKey;
 import org.opensearch.common.cache.LoadAwareCacheLoader;
 import org.opensearch.common.cache.RemovalListener;
 import org.opensearch.common.cache.RemovalNotification;
+import org.opensearch.common.cache.RemovalReason;
 import org.opensearch.common.cache.policy.CachedQueryResult;
 import org.opensearch.common.cache.serializer.BytesReferenceSerializer;
 import org.opensearch.common.cache.service.CacheService;
+import org.opensearch.common.cache.stats.ImmutableCacheStatsHolder;
 import org.opensearch.common.cache.store.config.CacheConfig;
+import org.opensearch.common.collect.Tuple;
 import org.opensearch.common.lease.Releasable;
 import org.opensearch.common.lucene.index.OpenSearchDirectoryReader;
 import org.opensearch.common.settings.Setting;
@@ -73,6 +78,7 @@
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
+import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
@@ -99,7 +105,7 @@
  *
  * @opensearch.internal
  */
-public final class IndicesRequestCache implements RemovalListener<IndicesRequestCache.Key, BytesReference>, Closeable {
+public final class IndicesRequestCache implements RemovalListener<ICacheKey<IndicesRequestCache.Key>, BytesReference>, Closeable {
 
     private static final Logger logger = LogManager.getLogger(IndicesRequestCache.class);
 
@@ -107,6 +113,10 @@ public final class IndicesRequestCache implements RemovalListener<IndicesRequest
      * A setting to enable or disable request caching on an index level. Its dynamic by default
      * since we are checking on the cluster state IndexMetadata always.
      */
+    public static final String INDICES_REQUEST_CACHE_CLEANUP_STALENESS_THRESHOLD_SETTING_KEY =
+        "indices.requests.cache.cleanup.staleness_threshold";
+    public static final String INDICES_REQUEST_CACHE_CLEANUP_INTERVAL_SETTING_KEY = "indices.requests.cache.cleanup.interval";
+
     public static final Setting<Boolean> INDEX_CACHE_REQUEST_ENABLED_SETTING = Setting.boolSetting(
         "index.requests.cache.enable",
         true,
@@ -123,15 +133,16 @@ public final class IndicesRequestCache implements RemovalListener<IndicesRequest
         new TimeValue(0),
         Property.NodeScope
     );
-    public static final Setting<TimeValue> INDICES_REQUEST_CACHE_CLEAN_INTERVAL_SETTING = Setting.positiveTimeSetting(
-        "indices.requests.cache.cleanup.interval",
+    public static final Setting<TimeValue> INDICES_REQUEST_CACHE_CLEANUP_INTERVAL_SETTING = Setting.positiveTimeSetting(
+        INDICES_REQUEST_CACHE_CLEANUP_INTERVAL_SETTING_KEY,
         INDICES_CACHE_CLEAN_INTERVAL_SETTING,
         Property.NodeScope
     );
     public static final Setting<String> INDICES_REQUEST_CACHE_STALENESS_THRESHOLD_SETTING = new Setting<>(
-        "indices.requests.cache.cleanup.staleness_threshold",
+        INDICES_REQUEST_CACHE_CLEANUP_STALENESS_THRESHOLD_SETTING_KEY,
         "0%",
         IndicesRequestCache::validateStalenessSetting,
+        Property.Dynamic,
         Property.NodeScope
     );
 
@@ -141,26 +152,35 @@ public final class IndicesRequestCache implements RemovalListener<IndicesRequest
     private final ByteSizeValue size;
     private final TimeValue expire;
     private final ICache<Key, BytesReference> cache;
+    private final ClusterService clusterService;
     private final Function<ShardId, Optional<CacheEntity>> cacheEntityLookup;
     // pkg-private for testing
     final IndicesRequestCacheCleanupManager cacheCleanupManager;
 
+    // These values determine the valid names for levels in the cache stats API
+    public static final String SHARD_ID_DIMENSION_NAME = "shards";
+    public static final String INDEX_DIMENSION_NAME = "indices";
+
     IndicesRequestCache(
         Settings settings,
         Function<ShardId, Optional<CacheEntity>> cacheEntityFunction,
         CacheService cacheService,
-        ThreadPool threadPool
+        ThreadPool threadPool,
+        ClusterService clusterService
     ) {
         this.size = INDICES_CACHE_QUERY_SIZE.get(settings);
         this.expire = INDICES_CACHE_QUERY_EXPIRE.exists(settings) ? INDICES_CACHE_QUERY_EXPIRE.get(settings) : null;
         long sizeInBytes = size.getBytes();
-        ToLongBiFunction<Key, BytesReference> weigher = (k, v) -> k.ramBytesUsed() + v.ramBytesUsed();
+        ToLongBiFunction<ICacheKey<Key>, BytesReference> weigher = (k, v) -> k.ramBytesUsed(k.key.ramBytesUsed()) + v.ramBytesUsed();
         this.cacheCleanupManager = new IndicesRequestCacheCleanupManager(
             threadPool,
-            INDICES_REQUEST_CACHE_CLEAN_INTERVAL_SETTING.get(settings),
+            INDICES_REQUEST_CACHE_CLEANUP_INTERVAL_SETTING.get(settings),
             getStalenessThreshold(settings)
         );
         this.cacheEntityLookup = cacheEntityFunction;
+        this.clusterService = clusterService;
+        this.clusterService.getClusterSettings()
+            .addSettingsUpdateConsumer(INDICES_REQUEST_CACHE_STALENESS_THRESHOLD_SETTING, this::setStalenessThreshold);
         this.cache = cacheService.createCache(
             new CacheConfig.Builder<Key, BytesReference>().setSettings(settings)
                 .setWeigher(weigher)
@@ -169,6 +189,7 @@ public final class IndicesRequestCache implements RemovalListener<IndicesRequest
                 .setRemovalListener(this)
                 .setMaxSizeInBytes(sizeInBytes) // for backward compatibility
                 .setExpireAfterAccess(expire) // for backward compatibility
+                .setDimensionNames(List.of(INDEX_DIMENSION_NAME, SHARD_ID_DIMENSION_NAME))
                 .setCachedResultParser((bytesReference) -> {
                     try {
                         return CachedQueryResult.getPolicyValues(bytesReference);
@@ -179,11 +200,17 @@ public final class IndicesRequestCache implements RemovalListener<IndicesRequest
                 })
                 .setKeySerializer(new IRCKeyWriteableSerializer())
                 .setValueSerializer(new BytesReferenceSerializer())
+                .setClusterSettings(clusterService.getClusterSettings())
                 .build(),
             CacheType.INDICES_REQUEST_CACHE
         );
     }
 
+    // package private for testing
+    void invalidateAll() {
+        cache.invalidateAll();
+    }
+
     @Override
     public void close() throws IOException {
         cache.invalidateAll();
@@ -196,20 +223,49 @@ private double getStalenessThreshold(Settings settings) {
         return RatioValue.parseRatioValue(threshold).getAsRatio();
     }
 
+    // pkg-private for testing
+    void setStalenessThreshold(String threshold) {
+        this.cacheCleanupManager.updateStalenessThreshold(RatioValue.parseRatioValue(threshold).getAsRatio());
+    }
+
     void clear(CacheEntity entity) {
         cacheCleanupManager.enqueueCleanupKey(new CleanupKey(entity, null));
         cacheCleanupManager.forceCleanCache();
     }
 
     @Override
-    public void onRemoval(RemovalNotification<Key, BytesReference> notification) {
+    public void onRemoval(RemovalNotification<ICacheKey<Key>, BytesReference> notification) {
         // In case this event happens for an old shard, we can safely ignore this as we don't keep track for old
         // shards as part of request cache.
-        Key key = notification.getKey();
-        cacheEntityLookup.apply(key.shardId).ifPresent(entity -> entity.onRemoval(notification));
-        cacheCleanupManager.updateCleanupKeyToCountMapOnCacheEviction(
-            new CleanupKey(cacheEntityLookup.apply(key.shardId).orElse(null), key.readerCacheKeyId)
-        );
+        // Pass a new removal notification containing Key rather than ICacheKey<Key> to the CacheEntity for backwards compatibility.
+        Key key = notification.getKey().key;
+        IndicesService.IndexShardCacheEntity indexShardCacheEntity = (IndicesService.IndexShardCacheEntity) cacheEntityLookup.apply(
+            key.shardId
+        ).orElse(null);
+        if (indexShardCacheEntity != null) {
+            // Here we match the hashcode to avoid scenario where we deduct stats of older IndexShard(with same
+            // shardId) from current IndexShard.
+            if (key.indexShardHashCode == System.identityHashCode(indexShardCacheEntity.getCacheIdentity())) {
+                indexShardCacheEntity.onRemoval(notification);
+            }
+        }
+        CleanupKey cleanupKey = new CleanupKey(indexShardCacheEntity, key.readerCacheKeyId);
+        cacheCleanupManager.updateStaleCountOnEntryRemoval(cleanupKey, notification);
+    }
+
+    private ICacheKey<Key> getICacheKey(Key key) {
+        String indexDimensionValue = getIndexDimensionName(key);
+        String shardIdDimensionValue = getShardIdDimensionName(key);
+        List<String> dimensions = List.of(indexDimensionValue, shardIdDimensionValue);
+        return new ICacheKey<>(key, dimensions);
+    }
+
+    private String getShardIdDimensionName(Key key) {
+        return key.shardId.toString();
+    }
+
+    private String getIndexDimensionName(Key key) {
+        return key.shardId.getIndexName();
     }
 
     BytesReference getOrCompute(
@@ -225,9 +281,10 @@ BytesReference getOrCompute(
             .getReaderCacheHelper();
         String readerCacheKeyId = delegatingCacheHelper.getDelegatingCacheKey().getId();
         assert readerCacheKeyId != null;
-        final Key key = new Key(((IndexShard) cacheEntity.getCacheIdentity()).shardId(), cacheKey, readerCacheKeyId);
+        IndexShard indexShard = ((IndexShard) cacheEntity.getCacheIdentity());
+        final Key key = new Key(indexShard.shardId(), cacheKey, readerCacheKeyId, System.identityHashCode(indexShard));
         Loader cacheLoader = new Loader(cacheEntity, loader);
-        BytesReference value = cache.computeIfAbsent(key, cacheLoader);
+        BytesReference value = cache.computeIfAbsent(getICacheKey(key), cacheLoader);
         if (cacheLoader.isLoaded()) {
             cacheEntity.onMiss();
             // see if it's the first time we see this reader, and make sure to register a cleanup key
@@ -238,7 +295,7 @@ BytesReference getOrCompute(
                     OpenSearchDirectoryReader.addReaderCloseListener(reader, cleanupKey);
                 }
             }
-            cacheCleanupManager.updateCleanupKeyToCountMapOnCacheInsertion(cleanupKey);
+            cacheCleanupManager.updateStaleCountOnCacheInsert(cleanupKey);
         } else {
             cacheEntity.onHit();
         }
@@ -258,7 +315,8 @@ void invalidate(IndicesService.IndexShardCacheEntity cacheEntity, DirectoryReade
             IndexReader.CacheHelper cacheHelper = ((OpenSearchDirectoryReader) reader).getDelegatingCacheHelper();
             readerCacheKeyId = ((OpenSearchDirectoryReader.DelegatingCacheHelper) cacheHelper).getDelegatingCacheKey().getId();
         }
-        cache.invalidate(new Key(((IndexShard) cacheEntity.getCacheIdentity()).shardId(), cacheKey, readerCacheKeyId));
+        IndexShard indexShard = (IndexShard) cacheEntity.getCacheIdentity();
+        cache.invalidate(getICacheKey(new Key(indexShard.shardId(), cacheKey, readerCacheKeyId, System.identityHashCode(indexShard))));
     }
 
     /**
@@ -266,7 +324,7 @@ void invalidate(IndicesService.IndexShardCacheEntity cacheEntity, DirectoryReade
      *
      * @opensearch.internal
      */
-    private static class Loader implements LoadAwareCacheLoader<Key, BytesReference> {
+    private static class Loader implements LoadAwareCacheLoader<ICacheKey<Key>, BytesReference> {
 
         private final CacheEntity entity;
         private final CheckedSupplier<BytesReference, IOException> loader;
@@ -282,7 +340,7 @@ public boolean isLoaded() {
         }
 
         @Override
-        public BytesReference load(Key key) throws Exception {
+        public BytesReference load(ICacheKey<Key> key) throws Exception {
             BytesReference value = loader.get();
             entity.onCached(key, value);
             loaded = true;
@@ -298,7 +356,7 @@ interface CacheEntity extends Accountable {
         /**
          * Called after the value was loaded.
          */
-        void onCached(Key key, BytesReference value);
+        void onCached(ICacheKey<Key> key, BytesReference value);
 
         /**
          * Returns <code>true</code> iff the resource behind this entity is still open ie.
@@ -325,7 +383,7 @@ interface CacheEntity extends Accountable {
         /**
          * Called when this entity instance is removed
          */
-        void onRemoval(RemovalNotification<Key, BytesReference> notification);
+        void onRemoval(RemovalNotification<ICacheKey<Key>, BytesReference> notification);
 
     }
 
@@ -336,19 +394,25 @@ interface CacheEntity extends Accountable {
      */
     static class Key implements Accountable, Writeable {
         public final ShardId shardId; // use as identity equality
+        public final int indexShardHashCode; // While ShardId is usually sufficient to uniquely identify an
+        // indexShard but in case where the same indexShard is deleted and reallocated on same node, we need the
+        // hashcode(default) to identify the older indexShard but with same shardId.
         public final String readerCacheKeyId;
         public final BytesReference value;
 
-        Key(ShardId shardId, BytesReference value, String readerCacheKeyId) {
+        Key(ShardId shardId, BytesReference value, String readerCacheKeyId, int indexShardHashCode) {
             this.shardId = shardId;
             this.value = value;
             this.readerCacheKeyId = Objects.requireNonNull(readerCacheKeyId);
+            this.indexShardHashCode = indexShardHashCode;
         }
 
         Key(StreamInput in) throws IOException {
             this.shardId = in.readOptionalWriteable(ShardId::new);
             this.readerCacheKeyId = in.readOptionalString();
             this.value = in.readBytesReference();
+            this.indexShardHashCode = in.readInt(); // We are serializing/de-serializing this as we need to store the
+            // key as part of tiered/disk cache. The key is not passed between nodes at this point.
         }
 
         @Override
@@ -370,6 +434,7 @@ public boolean equals(Object o) {
             if (!Objects.equals(readerCacheKeyId, key.readerCacheKeyId)) return false;
             if (!shardId.equals(key.shardId)) return false;
             if (!value.equals(key.value)) return false;
+            if (indexShardHashCode != key.indexShardHashCode) return false;
             return true;
         }
 
@@ -378,6 +443,7 @@ public int hashCode() {
             int result = shardId.hashCode();
             result = 31 * result + readerCacheKeyId.hashCode();
             result = 31 * result + value.hashCode();
+            result = 31 * result + indexShardHashCode;
             return result;
         }
 
@@ -386,6 +452,8 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeOptionalWriteable(shardId);
             out.writeOptionalString(readerCacheKeyId);
             out.writeBytesReference(value);
+            out.writeInt(indexShardHashCode); // We are serializing/de-serializing this as we need to store the
+            // key as part of tiered/disk cache. The key is not passed between nodes at this point.
         }
     }
 
@@ -440,7 +508,7 @@ class IndicesRequestCacheCleanupManager implements Closeable {
         private final Set<CleanupKey> keysToClean;
         private final ConcurrentMap<ShardId, HashMap<String, Integer>> cleanupKeyToCountMap;
         private final AtomicInteger staleKeysCount;
-        private final double stalenessThreshold;
+        private volatile double stalenessThreshold;
         private final IndicesRequestCacheCleaner cacheCleaner;
 
         IndicesRequestCacheCleanupManager(ThreadPool threadpool, TimeValue cleanInterval, double stalenessThreshold) {
@@ -452,6 +520,18 @@ class IndicesRequestCacheCleanupManager implements Closeable {
             threadpool.schedule(cacheCleaner, cleanInterval, ThreadPool.Names.SAME);
         }
 
+        void updateStalenessThreshold(double stalenessThreshold) {
+            double oldStalenessThreshold = this.stalenessThreshold;
+            this.stalenessThreshold = stalenessThreshold;
+            if (logger.isDebugEnabled()) {
+                logger.debug(
+                    "Staleness threshold for indices request cache changed to {} from {}",
+                    this.stalenessThreshold,
+                    oldStalenessThreshold
+                );
+            }
+        }
+
         /**
          * Enqueue cleanup key.
          *
@@ -474,8 +554,8 @@ void enqueueCleanupKey(CleanupKey cleanupKey) {
          *
          * @param cleanupKey the CleanupKey to be updated in the map
          */
-        private void updateCleanupKeyToCountMapOnCacheInsertion(CleanupKey cleanupKey) {
-            if (stalenessThreshold == 0.0 || cleanupKey.entity == null) {
+        private void updateStaleCountOnCacheInsert(CleanupKey cleanupKey) {
+            if (cleanupKey.entity == null) {
                 return;
             }
             IndexShard indexShard = (IndexShard) cleanupKey.entity.getCacheIdentity();
@@ -490,8 +570,32 @@ private void updateCleanupKeyToCountMapOnCacheInsertion(CleanupKey cleanupKey) {
             cleanupKeyToCountMap.computeIfAbsent(shardId, k -> new HashMap<>()).merge(cleanupKey.readerCacheKeyId, 1, Integer::sum);
         }
 
-        private void updateCleanupKeyToCountMapOnCacheEviction(CleanupKey cleanupKey) {
-            if (stalenessThreshold == 0.0 || cleanupKey.entity == null) {
+        /**
+         * Handles the eviction of a cache entry.
+         *
+         * <p>This method is called when an entry is evicted from the cache.
+         * We consider all removal notifications except with the reason Replaced
+         * {@link #incrementStaleKeysCount} would have removed the entries from the map and increment the {@link #staleKeysCount}
+         * Hence we decrement {@link #staleKeysCount} if we do not find the shardId or readerCacheKeyId in the map.
+         * Skip decrementing staleKeysCount if we find the shardId or readerCacheKeyId in the map since it would have not been accounted for in the staleKeysCount in
+         *
+         * @param cleanupKey   the CleanupKey that has been evicted from the cache
+         * @param notification RemovalNotification of the cache entry evicted
+         */
+        private void updateStaleCountOnEntryRemoval(
+            CleanupKey cleanupKey,
+            RemovalNotification<ICacheKey<Key>, BytesReference> notification
+        ) {
+            if (notification.getRemovalReason() == RemovalReason.REPLACED) {
+                // The reason of the notification is REPLACED when a cache entry's value is updated, since replacing an entry
+                // does not affect the staleness count, we skip such notifications.
+                return;
+            }
+            if (cleanupKey.entity == null) {
+                // entity will only be null when the shard is closed/deleted
+                // we would have accounted this in staleKeysCount when the closing/deletion of shard would have closed the associated
+                // readers
+                staleKeysCount.decrementAndGet();
                 return;
             }
             IndexShard indexShard = (IndexShard) cleanupKey.entity.getCacheIdentity();
@@ -501,15 +605,33 @@ private void updateCleanupKeyToCountMapOnCacheEviction(CleanupKey cleanupKey) {
             }
             ShardId shardId = indexShard.shardId();
 
-            cleanupKeyToCountMap.computeIfPresent(shardId, (shard, keyCountMap) -> {
-                keyCountMap.computeIfPresent(cleanupKey.readerCacheKeyId, (key, currentValue) -> {
-                    // decrement the stale key count
+            cleanupKeyToCountMap.compute(shardId, (key, readerCacheKeyMap) -> {
+                if (readerCacheKeyMap == null || !readerCacheKeyMap.containsKey(cleanupKey.readerCacheKeyId)) {
+                    // If ShardId is not present or readerCacheKeyId is not present
+                    // it should have already been accounted for and hence been removed from this map
+                    // so decrement staleKeysCount
                     staleKeysCount.decrementAndGet();
-                    int newValue = currentValue - 1;
-                    // Remove the key if the new value is zero by returning null; otherwise, update with the new value.
-                    return newValue == 0 ? null : newValue;
-                });
-                return keyCountMap;
+                    // Return the current map
+                    return readerCacheKeyMap;
+                } else {
+                    // If it is in the map, it is not stale yet.
+                    // Proceed to adjust the count for the readerCacheKeyId in the map
+                    // but do not decrement the staleKeysCount
+                    Integer count = readerCacheKeyMap.get(cleanupKey.readerCacheKeyId);
+                    // this should never be null
+                    assert (count != null && count >= 0);
+                    // Reduce the count by 1
+                    int newCount = count - 1;
+                    if (newCount > 0) {
+                        // Update the map with the new count
+                        readerCacheKeyMap.put(cleanupKey.readerCacheKeyId, newCount);
+                    } else {
+                        // Remove the readerCacheKeyId entry if new count is zero
+                        readerCacheKeyMap.remove(cleanupKey.readerCacheKeyId);
+                    }
+                    // If after modification, the readerCacheKeyMap is empty, we return null to remove the ShardId entry
+                    return readerCacheKeyMap.isEmpty() ? null : readerCacheKeyMap;
+                }
             });
         }
 
@@ -517,14 +639,14 @@ private void updateCleanupKeyToCountMapOnCacheEviction(CleanupKey cleanupKey) {
          * Updates the count of stale keys in the cache.
          * This method is called when a CleanupKey is added to the keysToClean set.
          *
-         * It increments the staleKeysCount by the count of the CleanupKey in the cleanupKeyToCountMap.
+         * <p>It increments the staleKeysCount by the count of the CleanupKey in the cleanupKeyToCountMap.
          * If the CleanupKey's readerCacheKeyId is null or the CleanupKey's entity is not open, it increments the staleKeysCount
          * by the total count of keys associated with the CleanupKey's ShardId in the cleanupKeyToCountMap and removes the ShardId from the map.
          *
          * @param cleanupKey the CleanupKey that has been marked for cleanup
          */
         private void incrementStaleKeysCount(CleanupKey cleanupKey) {
-            if (stalenessThreshold == 0.0 || cleanupKey.entity == null) {
+            if (cleanupKey.entity == null) {
                 return;
             }
             IndexShard indexShard = (IndexShard) cleanupKey.entity.getCacheIdentity();
@@ -535,7 +657,7 @@ private void incrementStaleKeysCount(CleanupKey cleanupKey) {
             ShardId shardId = indexShard.shardId();
 
             // Using computeIfPresent to atomically operate on the countMap for a given shardId
-            cleanupKeyToCountMap.computeIfPresent(shardId, (key, countMap) -> {
+            cleanupKeyToCountMap.computeIfPresent(shardId, (currentShardId, countMap) -> {
                 if (cleanupKey.readerCacheKeyId == null) {
                     // Aggregate and add to staleKeysCount atomically if readerCacheKeyId is null
                     int totalSum = countMap.values().stream().mapToInt(Integer::intValue).sum();
@@ -544,18 +666,19 @@ private void incrementStaleKeysCount(CleanupKey cleanupKey) {
                     return null;
                 } else {
                     // Update staleKeysCount based on specific readerCacheKeyId, then remove it from the countMap
-                    countMap.computeIfPresent(cleanupKey.readerCacheKeyId, (k, v) -> {
-                        staleKeysCount.addAndGet(v);
+                    countMap.computeIfPresent(cleanupKey.readerCacheKeyId, (readerCacheKey, count) -> {
+                        staleKeysCount.addAndGet(count);
                         // Return null to remove the key after updating staleKeysCount
                         return null;
                     });
-
                     // Check if countMap is empty after removal to decide if we need to remove the shardId entry
                     if (countMap.isEmpty()) {
-                        return null; // Returning null removes the entry for shardId
+                        // Returning null removes the entry for shardId
+                        return null;
                     }
                 }
-                return countMap; // Return the modified countMap to keep the mapping
+                // Return the modified countMap to retain updates
+                return countMap;
             });
         }
 
@@ -593,14 +716,16 @@ private synchronized void cleanCache(double stalenessThreshold) {
             // Contains CleanupKey objects with open shard but invalidated readerCacheKeyId.
             final Set<CleanupKey> cleanupKeysFromOutdatedReaders = new HashSet<>();
             // Contains CleanupKey objects of a closed shard.
-            final Set<Object> cleanupKeysFromClosedShards = new HashSet<>();
+            final Set<Tuple<ShardId, Integer>> cleanupKeysFromClosedShards = new HashSet<>();
 
             for (Iterator<CleanupKey> iterator = keysToClean.iterator(); iterator.hasNext();) {
                 CleanupKey cleanupKey = iterator.next();
                 iterator.remove();
                 if (cleanupKey.readerCacheKeyId == null || !cleanupKey.entity.isOpen()) {
                     // null indicates full cleanup, as does a closed shard
-                    cleanupKeysFromClosedShards.add(((IndexShard) cleanupKey.entity.getCacheIdentity()).shardId());
+                    IndexShard indexShard = (IndexShard) cleanupKey.entity.getCacheIdentity();
+                    // Add both shardId and indexShardHashCode to uniquely identify an indexShard.
+                    cleanupKeysFromClosedShards.add(new Tuple<>(indexShard.shardId(), indexShard.hashCode()));
                 } else {
                     cleanupKeysFromOutdatedReaders.add(cleanupKey);
                 }
@@ -610,17 +735,35 @@ private synchronized void cleanCache(double stalenessThreshold) {
                 return;
             }
 
-            for (Iterator<Key> iterator = cache.keys().iterator(); iterator.hasNext();) {
-                Key key = iterator.next();
-                if (cleanupKeysFromClosedShards.contains(key.shardId)) {
+            Set<List<String>> dimensionListsToDrop = new HashSet<>();
+
+            for (Iterator<ICacheKey<Key>> iterator = cache.keys().iterator(); iterator.hasNext();) {
+                ICacheKey<Key> key = iterator.next();
+                Key delegatingKey = key.key;
+                if (cleanupKeysFromClosedShards.contains(new Tuple<>(delegatingKey.shardId, delegatingKey.indexShardHashCode))) {
+                    // Since the shard is closed, the cache should drop stats for this shard.
+                    dimensionListsToDrop.add(key.dimensions);
                     iterator.remove();
                 } else {
-                    CleanupKey cleanupKey = new CleanupKey(cacheEntityLookup.apply(key.shardId).orElse(null), key.readerCacheKeyId);
-                    if (cleanupKeysFromOutdatedReaders.contains(cleanupKey)) {
+                    CacheEntity cacheEntity = cacheEntityLookup.apply(delegatingKey.shardId).orElse(null);
+                    if (cacheEntity == null) {
+                        // If cache entity is null, it means that index or shard got deleted/closed meanwhile.
+                        // So we will delete this key.
                         iterator.remove();
+                    } else {
+                        CleanupKey cleanupKey = new CleanupKey(cacheEntity, delegatingKey.readerCacheKeyId);
+                        if (cleanupKeysFromOutdatedReaders.contains(cleanupKey)) {
+                            iterator.remove();
+                        }
                     }
                 }
             }
+            for (List<String> closedDimensions : dimensionListsToDrop) {
+                // Invalidate a dummy key containing the dimensions we need to drop stats for
+                ICacheKey<Key> dummyKey = new ICacheKey<>(null, closedDimensions);
+                dummyKey.setDropStatsForDimensions(true);
+                cache.invalidate(dummyKey);
+            }
             cache.refresh();
         }
 
@@ -670,6 +813,11 @@ public void close() {
             this.cacheCleaner.close();
         }
 
+        // for testing
+        ConcurrentMap<ShardId, HashMap<String, Integer>> getCleanupKeyToCountMap() {
+            return cleanupKeyToCountMap;
+        }
+
         private final class IndicesRequestCacheCleaner implements Runnable, Releasable {
 
             private final IndicesRequestCacheCleanupManager cacheCleanupManager;
@@ -711,6 +859,13 @@ long count() {
         return cache.count();
     }
 
+    /**
+     * Returns the current cache stats. Pkg-private for testing.
+     */
+    ImmutableCacheStatsHolder stats(String[] levels) {
+        return cache.stats(levels);
+    }
+
     int numRegisteredCloseListeners() { // for testing
         return registeredClosedListeners.size();
     }
diff --git a/server/src/main/java/org/opensearch/indices/IndicesService.java b/server/src/main/java/org/opensearch/indices/IndicesService.java
index 9bc81c1826c2d..251be8a990055 100644
--- a/server/src/main/java/org/opensearch/indices/IndicesService.java
+++ b/server/src/main/java/org/opensearch/indices/IndicesService.java
@@ -54,6 +54,7 @@
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
 import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.cluster.node.DiscoveryNodes;
 import org.opensearch.cluster.routing.RecoverySource;
 import org.opensearch.cluster.routing.ShardRouting;
 import org.opensearch.cluster.service.ClusterService;
@@ -123,7 +124,6 @@
 import org.opensearch.index.query.QueryRewriteContext;
 import org.opensearch.index.recovery.RecoveryStats;
 import org.opensearch.index.refresh.RefreshStats;
-import org.opensearch.index.remote.RemoteStorePathType;
 import org.opensearch.index.remote.RemoteStoreStatsTrackerFactory;
 import org.opensearch.index.search.stats.SearchStats;
 import org.opensearch.index.seqno.RetentionLeaseStats;
@@ -216,10 +216,11 @@ public class IndicesService extends AbstractLifecycleComponent
         IndicesClusterStateService.AllocatedIndices<IndexShard, IndexService>,
         IndexService.ShardStoreDeleter {
     private static final Logger logger = LogManager.getLogger(IndicesService.class);
+    public static final String INDICES_CACHE_CLEANUP_INTERVAL_SETTING_KEY = "indices.cache.cleanup_interval";
 
     public static final String INDICES_SHARDS_CLOSED_TIMEOUT = "indices.shards_closed_timeout";
     public static final Setting<TimeValue> INDICES_CACHE_CLEAN_INTERVAL_SETTING = Setting.positiveTimeSetting(
-        "indices.cache.cleanup_interval",
+        INDICES_CACHE_CLEANUP_INTERVAL_SETTING_KEY,
         TimeValue.timeValueMinutes(1),
         Property.NodeScope
     );
@@ -249,17 +250,6 @@ public class IndicesService extends AbstractLifecycleComponent
         Property.Final
     );
 
-    /**
-     * Used to specify the default translog buffer interval for remote store backed indexes.
-     */
-    public static final Setting<TimeValue> CLUSTER_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING = Setting.timeSetting(
-        "cluster.remote_store.translog.buffer_interval",
-        IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL,
-        IndexSettings.MINIMUM_REMOTE_TRANSLOG_BUFFER_INTERVAL,
-        Property.NodeScope,
-        Property.Dynamic
-    );
-
     /**
      * This setting is used to set the refresh interval when the {@code index.refresh_interval} index setting is not
      * provided during index creation or when the existing {@code index.refresh_interval} index setting is set as null.
@@ -284,8 +274,8 @@ public class IndicesService extends AbstractLifecycleComponent
      */
     public static final Setting<TimeValue> CLUSTER_MINIMUM_INDEX_REFRESH_INTERVAL_SETTING = Setting.timeSetting(
         "cluster.minimum.index.refresh_interval",
-        IndexSettings.MINIMUM_REFRESH_INTERVAL,
-        IndexSettings.MINIMUM_REFRESH_INTERVAL,
+        TimeValue.ZERO,
+        TimeValue.ZERO,
         new ClusterMinimumRefreshIntervalValidator(),
         Property.NodeScope,
         Property.Dynamic
@@ -316,18 +306,6 @@ public class IndicesService extends AbstractLifecycleComponent
         Property.Final
     );
 
-    /**
-     * This setting is used to set the remote store blob store path prefix strategy. This setting is effective only for
-     * remote store enabled cluster.
-     */
-    public static final Setting<RemoteStorePathType> CLUSTER_REMOTE_STORE_PATH_PREFIX_TYPE_SETTING = new Setting<>(
-        "cluster.remote_store.index.path.prefix.type",
-        RemoteStorePathType.FIXED.toString(),
-        RemoteStorePathType::parseString,
-        Property.NodeScope,
-        Property.Dynamic
-    );
-
     /**
      * The node's settings.
      */
@@ -366,7 +344,7 @@ public class IndicesService extends AbstractLifecycleComponent
     private volatile boolean idFieldDataEnabled;
     private volatile boolean allowExpensiveQueries;
     private final RecoverySettings recoverySettings;
-
+    private final RemoteStoreSettings remoteStoreSettings;
     @Nullable
     private final OpenSearchThreadPoolExecutor danglingIndicesThreadPoolExecutor;
     private final Set<Index> danglingIndicesToWrite = Sets.newConcurrentHashSet();
@@ -375,8 +353,6 @@ public class IndicesService extends AbstractLifecycleComponent
     private final IndexStorePlugin.DirectoryFactory remoteDirectoryFactory;
     private final BiFunction<IndexSettings, ShardRouting, TranslogFactory> translogFactorySupplier;
     private volatile TimeValue clusterDefaultRefreshInterval;
-    private volatile TimeValue clusterRemoteTranslogBufferInterval;
-
     private final SearchRequestStats searchRequestStats;
 
     @Override
@@ -411,7 +387,8 @@ public IndicesService(
         SearchRequestStats searchRequestStats,
         @Nullable RemoteStoreStatsTrackerFactory remoteStoreStatsTrackerFactory,
         RecoverySettings recoverySettings,
-        CacheService cacheService
+        CacheService cacheService,
+        RemoteStoreSettings remoteStoreSettings
     ) {
         this.settings = settings;
         this.threadPool = threadPool;
@@ -428,7 +405,7 @@ public IndicesService(
                 return Optional.empty();
             }
             return Optional.of(new IndexShardCacheEntity(indexService.getShard(shardId.id())));
-        }), cacheService, threadPool);
+        }), cacheService, threadPool, clusterService);
         this.indicesQueryCache = new IndicesQueryCache(settings);
         this.mapperRegistry = mapperRegistry;
         this.namedWriteableRegistry = namedWriteableRegistry;
@@ -509,16 +486,15 @@ protected void closeInternal() {
             repositoriesServiceSupplier,
             threadPool,
             remoteStoreStatsTrackerFactory,
-            settings
+            settings,
+            remoteStoreSettings
         );
         this.searchRequestStats = searchRequestStats;
         this.clusterDefaultRefreshInterval = CLUSTER_DEFAULT_INDEX_REFRESH_INTERVAL_SETTING.get(clusterService.getSettings());
         clusterService.getClusterSettings()
             .addSettingsUpdateConsumer(CLUSTER_DEFAULT_INDEX_REFRESH_INTERVAL_SETTING, this::onRefreshIntervalUpdate);
-        this.clusterRemoteTranslogBufferInterval = CLUSTER_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.get(clusterService.getSettings());
-        clusterService.getClusterSettings()
-            .addSettingsUpdateConsumer(CLUSTER_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING, this::setClusterRemoteTranslogBufferInterval);
         this.recoverySettings = recoverySettings;
+        this.remoteStoreSettings = remoteStoreSettings;
     }
 
     /**
@@ -541,7 +517,8 @@ private static BiFunction<IndexSettings, ShardRouting, TranslogFactory> getTrans
         Supplier<RepositoriesService> repositoriesServiceSupplier,
         ThreadPool threadPool,
         RemoteStoreStatsTrackerFactory remoteStoreStatsTrackerFactory,
-        Settings settings
+        Settings settings,
+        RemoteStoreSettings remoteStoreSettings
     ) {
         return (indexSettings, shardRouting) -> {
             if (indexSettings.isRemoteTranslogStoreEnabled() && shardRouting.primary()) {
@@ -549,14 +526,16 @@ private static BiFunction<IndexSettings, ShardRouting, TranslogFactory> getTrans
                     repositoriesServiceSupplier,
                     threadPool,
                     indexSettings.getRemoteStoreTranslogRepository(),
-                    remoteStoreStatsTrackerFactory.getRemoteTranslogTransferTracker(shardRouting.shardId())
+                    remoteStoreStatsTrackerFactory.getRemoteTranslogTransferTracker(shardRouting.shardId()),
+                    remoteStoreSettings
                 );
             } else if (isRemoteDataAttributePresent(settings) && shardRouting.primary()) {
                 return new RemoteBlobStoreInternalTranslogFactory(
                     repositoriesServiceSupplier,
                     threadPool,
                     RemoteStoreNodeAttribute.getRemoteStoreTranslogRepo(indexSettings.getNodeSettings()),
-                    remoteStoreStatsTrackerFactory.getRemoteTranslogTransferTracker(shardRouting.shardId())
+                    remoteStoreStatsTrackerFactory.getRemoteTranslogTransferTracker(shardRouting.shardId()),
+                    remoteStoreSettings
                 );
             }
             return new InternalTranslogFactory();
@@ -923,8 +902,8 @@ private synchronized IndexService createIndexService(
             remoteDirectoryFactory,
             translogFactorySupplier,
             this::getClusterDefaultRefreshInterval,
-            this::getClusterRemoteTranslogBufferInterval,
-            this.recoverySettings
+            this.recoverySettings,
+            this.remoteStoreSettings
         );
     }
 
@@ -947,7 +926,7 @@ private EngineFactory getEngineFactory(final IndexSettings idxSettings) {
             if (idxSettings.isRemoteSnapshot()) {
                 return config -> new ReadOnlyEngine(config, new SeqNoStats(0, 0, 0), new TranslogStats(), true, Function.identity(), false);
             }
-            if (idxSettings.isSegRepEnabledOrRemoteNode() || idxSettings.isRemoteNode()) {
+            if (idxSettings.isSegRepEnabledOrRemoteNode() || idxSettings.isAssignedOnRemoteNode()) {
                 return new NRTReplicationEngineFactory();
             }
             return new InternalEngineFactory();
@@ -1035,7 +1014,8 @@ public IndexShard createShard(
         final RetentionLeaseSyncer retentionLeaseSyncer,
         final DiscoveryNode targetNode,
         final DiscoveryNode sourceNode,
-        final RemoteStoreStatsTrackerFactory remoteStoreStatsTrackerFactory
+        final RemoteStoreStatsTrackerFactory remoteStoreStatsTrackerFactory,
+        final DiscoveryNodes discoveryNodes
     ) throws IOException {
         Objects.requireNonNull(retentionLeaseSyncer);
         ensureChangesAllowed();
@@ -1050,7 +1030,8 @@ public IndexShard createShard(
             remoteStoreStatsTrackerFactory,
             repositoriesService,
             targetNode,
-            sourceNode
+            sourceNode,
+            discoveryNodes
         );
         indexShard.addShardFailureCallback(onShardFailure);
         indexShard.startRecovery(recoveryState, recoveryTargetService, recoveryListener, repositoriesService, mapping -> {
@@ -2029,6 +2010,9 @@ public Iterator<Setting<?>> settings() {
      * @param defaultRefreshInterval value of cluster default index refresh interval setting
      */
     private static void validateRefreshIntervalSettings(TimeValue minimumRefreshInterval, TimeValue defaultRefreshInterval) {
+        if (defaultRefreshInterval.millis() < 0) {
+            return;
+        }
         if (minimumRefreshInterval.compareTo(defaultRefreshInterval) > 0) {
             throw new IllegalArgumentException(
                 "cluster minimum index refresh interval ["
@@ -2044,12 +2028,7 @@ private TimeValue getClusterDefaultRefreshInterval() {
         return this.clusterDefaultRefreshInterval;
     }
 
-    // Exclusively for testing, please do not use it elsewhere.
-    public TimeValue getClusterRemoteTranslogBufferInterval() {
-        return clusterRemoteTranslogBufferInterval;
-    }
-
-    private void setClusterRemoteTranslogBufferInterval(TimeValue clusterRemoteTranslogBufferInterval) {
-        this.clusterRemoteTranslogBufferInterval = clusterRemoteTranslogBufferInterval;
+    public RemoteStoreSettings getRemoteStoreSettings() {
+        return this.remoteStoreSettings;
     }
 }
diff --git a/server/src/main/java/org/opensearch/indices/RemoteStoreSettings.java b/server/src/main/java/org/opensearch/indices/RemoteStoreSettings.java
new file mode 100644
index 0000000000000..0bd4c7aedfc03
--- /dev/null
+++ b/server/src/main/java/org/opensearch/indices/RemoteStoreSettings.java
@@ -0,0 +1,193 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.indices;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.common.annotation.PublicApi;
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Setting;
+import org.opensearch.common.settings.Setting.Property;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.TimeValue;
+import org.opensearch.index.IndexSettings;
+import org.opensearch.index.remote.RemoteStoreEnums;
+
+/**
+ * Settings for remote store
+ *
+ * @opensearch.api
+ */
+@PublicApi(since = "2.14.0")
+public class RemoteStoreSettings {
+
+    /**
+     * Used to specify the default translog buffer interval for remote store backed indexes.
+     */
+    public static final Setting<TimeValue> CLUSTER_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING = Setting.timeSetting(
+        "cluster.remote_store.translog.buffer_interval",
+        IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL,
+        IndexSettings.MINIMUM_REMOTE_TRANSLOG_BUFFER_INTERVAL,
+        Property.NodeScope,
+        Property.Dynamic
+    );
+
+    /**
+     * Controls minimum number of metadata files to keep in remote segment store.
+     * {@code value < 1} will disable deletion of stale segment metadata files.
+     */
+    public static final Setting<Integer> CLUSTER_REMOTE_INDEX_SEGMENT_METADATA_RETENTION_MAX_COUNT_SETTING = Setting.intSetting(
+        "cluster.remote_store.index.segment_metadata.retention.max_count",
+        10,
+        -1,
+        v -> {
+            if (v == 0) {
+                throw new IllegalArgumentException(
+                    "Value 0 is not allowed for this setting as it would delete all the data from remote segment store"
+                );
+            }
+        },
+        Property.NodeScope,
+        Property.Dynamic
+    );
+
+    /**
+     * Controls timeout value while uploading translog and checkpoint files to remote translog
+     */
+    public static final Setting<TimeValue> CLUSTER_REMOTE_TRANSLOG_TRANSFER_TIMEOUT_SETTING = Setting.timeSetting(
+        "cluster.remote_store.translog.transfer_timeout",
+        TimeValue.timeValueSeconds(30),
+        TimeValue.timeValueSeconds(30),
+        Property.NodeScope,
+        Property.Dynamic
+    );
+
+    /**
+     * This setting is used to set the remote store blob store path type strategy. This setting is effective only for
+     * remote store enabled cluster.
+     */
+    @ExperimentalApi
+    public static final Setting<RemoteStoreEnums.PathType> CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING = new Setting<>(
+        "cluster.remote_store.index.path.type",
+        RemoteStoreEnums.PathType.FIXED.toString(),
+        RemoteStoreEnums.PathType::parseString,
+        Property.NodeScope,
+        Property.Dynamic
+    );
+
+    /**
+     * This setting is used to set the remote store blob store path hash algorithm strategy. This setting is effective only for
+     * remote store enabled cluster. This setting will come to effect if the {@link #CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING}
+     * is either {@code HASHED_PREFIX} or {@code HASHED_INFIX}.
+     */
+    @ExperimentalApi
+    public static final Setting<RemoteStoreEnums.PathHashAlgorithm> CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING = new Setting<>(
+        "cluster.remote_store.index.path.hash_algorithm",
+        RemoteStoreEnums.PathHashAlgorithm.FNV_1A_COMPOSITE_1.toString(),
+        RemoteStoreEnums.PathHashAlgorithm::parseString,
+        Property.NodeScope,
+        Property.Dynamic
+    );
+
+    /**
+     * Controls the maximum referenced remote translog files. If breached the shard will be flushed.
+     */
+    public static final Setting<Integer> CLUSTER_REMOTE_MAX_TRANSLOG_READERS = Setting.intSetting(
+        "cluster.remote_store.translog.max_readers",
+        1000,
+        100,
+        Property.Dynamic,
+        Property.NodeScope
+    );
+
+    private volatile TimeValue clusterRemoteTranslogBufferInterval;
+    private volatile int minRemoteSegmentMetadataFiles;
+    private volatile TimeValue clusterRemoteTranslogTransferTimeout;
+    private volatile RemoteStoreEnums.PathType pathType;
+    private volatile RemoteStoreEnums.PathHashAlgorithm pathHashAlgorithm;
+    private volatile int maxRemoteTranslogReaders;
+
+    public RemoteStoreSettings(Settings settings, ClusterSettings clusterSettings) {
+        clusterRemoteTranslogBufferInterval = CLUSTER_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.get(settings);
+        clusterSettings.addSettingsUpdateConsumer(
+            CLUSTER_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING,
+            this::setClusterRemoteTranslogBufferInterval
+        );
+
+        minRemoteSegmentMetadataFiles = CLUSTER_REMOTE_INDEX_SEGMENT_METADATA_RETENTION_MAX_COUNT_SETTING.get(settings);
+        clusterSettings.addSettingsUpdateConsumer(
+            CLUSTER_REMOTE_INDEX_SEGMENT_METADATA_RETENTION_MAX_COUNT_SETTING,
+            this::setMinRemoteSegmentMetadataFiles
+        );
+
+        clusterRemoteTranslogTransferTimeout = CLUSTER_REMOTE_TRANSLOG_TRANSFER_TIMEOUT_SETTING.get(settings);
+        clusterSettings.addSettingsUpdateConsumer(
+            CLUSTER_REMOTE_TRANSLOG_TRANSFER_TIMEOUT_SETTING,
+            this::setClusterRemoteTranslogTransferTimeout
+        );
+
+        pathType = clusterSettings.get(CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING);
+        clusterSettings.addSettingsUpdateConsumer(CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING, this::setPathType);
+
+        pathHashAlgorithm = clusterSettings.get(CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING);
+        clusterSettings.addSettingsUpdateConsumer(CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING, this::setPathHashAlgorithm);
+
+        maxRemoteTranslogReaders = CLUSTER_REMOTE_MAX_TRANSLOG_READERS.get(settings);
+        clusterSettings.addSettingsUpdateConsumer(CLUSTER_REMOTE_MAX_TRANSLOG_READERS, this::setMaxRemoteTranslogReaders);
+    }
+
+    public TimeValue getClusterRemoteTranslogBufferInterval() {
+        return clusterRemoteTranslogBufferInterval;
+    }
+
+    private void setClusterRemoteTranslogBufferInterval(TimeValue clusterRemoteTranslogBufferInterval) {
+        this.clusterRemoteTranslogBufferInterval = clusterRemoteTranslogBufferInterval;
+    }
+
+    private void setMinRemoteSegmentMetadataFiles(int minRemoteSegmentMetadataFiles) {
+        this.minRemoteSegmentMetadataFiles = minRemoteSegmentMetadataFiles;
+    }
+
+    public int getMinRemoteSegmentMetadataFiles() {
+        return this.minRemoteSegmentMetadataFiles;
+    }
+
+    public TimeValue getClusterRemoteTranslogTransferTimeout() {
+        return clusterRemoteTranslogTransferTimeout;
+    }
+
+    private void setClusterRemoteTranslogTransferTimeout(TimeValue clusterRemoteTranslogTransferTimeout) {
+        this.clusterRemoteTranslogTransferTimeout = clusterRemoteTranslogTransferTimeout;
+    }
+
+    @ExperimentalApi
+    public RemoteStoreEnums.PathType getPathType() {
+        return pathType;
+    }
+
+    @ExperimentalApi
+    public RemoteStoreEnums.PathHashAlgorithm getPathHashAlgorithm() {
+        return pathHashAlgorithm;
+    }
+
+    private void setPathType(RemoteStoreEnums.PathType pathType) {
+        this.pathType = pathType;
+    }
+
+    private void setPathHashAlgorithm(RemoteStoreEnums.PathHashAlgorithm pathHashAlgorithm) {
+        this.pathHashAlgorithm = pathHashAlgorithm;
+    }
+
+    public int getMaxRemoteTranslogReaders() {
+        return maxRemoteTranslogReaders;
+    }
+
+    private void setMaxRemoteTranslogReaders(int maxRemoteTranslogReaders) {
+        this.maxRemoteTranslogReaders = maxRemoteTranslogReaders;
+    }
+}
diff --git a/server/src/main/java/org/opensearch/indices/cluster/IndicesClusterStateService.java b/server/src/main/java/org/opensearch/indices/cluster/IndicesClusterStateService.java
index 7fb8b172ae352..2c3ffcdd9e0ba 100644
--- a/server/src/main/java/org/opensearch/indices/cluster/IndicesClusterStateService.java
+++ b/server/src/main/java/org/opensearch/indices/cluster/IndicesClusterStateService.java
@@ -680,7 +680,8 @@ private void createShard(DiscoveryNodes nodes, RoutingTable routingTable, ShardR
                 retentionLeaseSyncer,
                 nodes.getLocalNode(),
                 sourceNode,
-                remoteStoreStatsTrackerFactory
+                remoteStoreStatsTrackerFactory,
+                nodes
             );
         } catch (Exception e) {
             failAndRemoveShard(shardRouting, true, "failed to create shard", e, state);
@@ -714,7 +715,8 @@ private void updateShard(
                 primaryReplicaSyncer::resync,
                 clusterState.version(),
                 inSyncIds,
-                indexShardRoutingTable
+                indexShardRoutingTable,
+                nodes
             );
         } catch (Exception e) {
             failAndRemoveShard(shardRouting, true, "failed updating shard routing entry", e, clusterState);
@@ -922,7 +924,8 @@ void updateShardState(
             BiConsumer<IndexShard, ActionListener<ResyncTask>> primaryReplicaSyncer,
             long applyingClusterStateVersion,
             Set<String> inSyncAllocationIds,
-            IndexShardRoutingTable routingTable
+            IndexShardRoutingTable routingTable,
+            DiscoveryNodes discoveryNodes
         ) throws IOException;
     }
 
@@ -1040,7 +1043,8 @@ T createShard(
             RetentionLeaseSyncer retentionLeaseSyncer,
             DiscoveryNode targetNode,
             @Nullable DiscoveryNode sourceNode,
-            RemoteStoreStatsTrackerFactory remoteStoreStatsTrackerFactory
+            RemoteStoreStatsTrackerFactory remoteStoreStatsTrackerFactory,
+            DiscoveryNodes discoveryNodes
         ) throws IOException;
 
         /**
diff --git a/server/src/main/java/org/opensearch/indices/recovery/PeerRecoveryTargetService.java b/server/src/main/java/org/opensearch/indices/recovery/PeerRecoveryTargetService.java
index 227496f72f83d..6279a8ec3646c 100644
--- a/server/src/main/java/org/opensearch/indices/recovery/PeerRecoveryTargetService.java
+++ b/server/src/main/java/org/opensearch/indices/recovery/PeerRecoveryTargetService.java
@@ -261,7 +261,7 @@ private void doRecovery(final long recoveryId, final StartRecoveryRequest preExi
                         }
                     }
                     final boolean hasRemoteTranslog = recoveryTarget.state().getPrimary() == false
-                        && indexShard.indexSettings().isRemoteNode();
+                        && indexShard.indexSettings().isAssignedOnRemoteNode();
                     final boolean hasNoTranslog = indexShard.indexSettings().isRemoteSnapshot();
                     final boolean verifyTranslog = (hasRemoteTranslog || hasNoTranslog || hasRemoteSegmentStore) == false;
                     final long startingSeqNo = indexShard.recoverLocallyAndFetchStartSeqNo(!hasRemoteTranslog);
@@ -575,7 +575,13 @@ public void messageReceived(final FileChunkRequest request, TransportChannel cha
             try (ReplicationRef<RecoveryTarget> recoveryRef = onGoingRecoveries.getSafe(request.recoveryId(), request.shardId())) {
                 final RecoveryTarget recoveryTarget = recoveryRef.get();
                 final ActionListener<Void> listener = recoveryTarget.createOrFinishListener(channel, Actions.FILE_CHUNK, request);
-                recoveryTarget.handleFileChunk(request, recoveryTarget, bytesSinceLastPause, recoverySettings.rateLimiter(), listener);
+                recoveryTarget.handleFileChunk(
+                    request,
+                    recoveryTarget,
+                    bytesSinceLastPause,
+                    recoverySettings.recoveryRateLimiter(),
+                    listener
+                );
             }
         }
     }
diff --git a/server/src/main/java/org/opensearch/indices/recovery/RecoverySettings.java b/server/src/main/java/org/opensearch/indices/recovery/RecoverySettings.java
index 2b41eb125d808..8f9da6babdd99 100644
--- a/server/src/main/java/org/opensearch/indices/recovery/RecoverySettings.java
+++ b/server/src/main/java/org/opensearch/indices/recovery/RecoverySettings.java
@@ -65,6 +65,16 @@ public class RecoverySettings {
         Property.NodeScope
     );
 
+    /**
+     * Individual speed setting for segment replication, default -1B to reuse the setting of recovery.
+     */
+    public static final Setting<ByteSizeValue> INDICES_REPLICATION_MAX_BYTES_PER_SEC_SETTING = Setting.byteSizeSetting(
+        "indices.replication.max_bytes_per_sec",
+        new ByteSizeValue(-1),
+        Property.Dynamic,
+        Property.NodeScope
+    );
+
     /**
      * Controls the maximum number of file chunk requests that can be sent concurrently from the source node to the target node.
      */
@@ -159,25 +169,6 @@ public class RecoverySettings {
         Property.NodeScope
     );
 
-    /**
-     * Controls minimum number of metadata files to keep in remote segment store.
-     * {@code value < 1} will disable deletion of stale segment metadata files.
-     */
-    public static final Setting<Integer> CLUSTER_REMOTE_INDEX_SEGMENT_METADATA_RETENTION_MAX_COUNT_SETTING = Setting.intSetting(
-        "cluster.remote_store.index.segment_metadata.retention.max_count",
-        10,
-        -1,
-        v -> {
-            if (v == 0) {
-                throw new IllegalArgumentException(
-                    "Value 0 is not allowed for this setting as it would delete all the data from remote segment store"
-                );
-            }
-        },
-        Property.NodeScope,
-        Property.Dynamic
-    );
-
     public static final Setting<TimeValue> INDICES_INTERNAL_REMOTE_UPLOAD_TIMEOUT = Setting.timeSetting(
         "indices.recovery.internal_remote_upload_timeout",
         new TimeValue(1, TimeUnit.HOURS),
@@ -188,18 +179,19 @@ public class RecoverySettings {
     // choose 512KB-16B to ensure that the resulting byte[] is not a humongous allocation in G1.
     public static final ByteSizeValue DEFAULT_CHUNK_SIZE = new ByteSizeValue(512 * 1024 - 16, ByteSizeUnit.BYTES);
 
-    private volatile ByteSizeValue maxBytesPerSec;
+    private volatile ByteSizeValue recoveryMaxBytesPerSec;
+    private volatile ByteSizeValue replicationMaxBytesPerSec;
     private volatile int maxConcurrentFileChunks;
     private volatile int maxConcurrentOperations;
     private volatile int maxConcurrentRemoteStoreStreams;
-    private volatile SimpleRateLimiter rateLimiter;
+    private volatile SimpleRateLimiter recoveryRateLimiter;
+    private volatile SimpleRateLimiter replicationRateLimiter;
     private volatile TimeValue retryDelayStateSync;
     private volatile TimeValue retryDelayNetwork;
     private volatile TimeValue activityTimeout;
     private volatile TimeValue internalActionTimeout;
     private volatile TimeValue internalActionRetryTimeout;
     private volatile TimeValue internalActionLongTimeout;
-    private volatile int minRemoteSegmentMetadataFiles;
 
     private volatile ByteSizeValue chunkSize = DEFAULT_CHUNK_SIZE;
     private volatile TimeValue internalRemoteUploadTimeout;
@@ -218,17 +210,20 @@ public RecoverySettings(Settings settings, ClusterSettings clusterSettings) {
         this.internalActionLongTimeout = INDICES_RECOVERY_INTERNAL_LONG_ACTION_TIMEOUT_SETTING.get(settings);
 
         this.activityTimeout = INDICES_RECOVERY_ACTIVITY_TIMEOUT_SETTING.get(settings);
-        this.maxBytesPerSec = INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.get(settings);
-        if (maxBytesPerSec.getBytes() <= 0) {
-            rateLimiter = null;
+        this.recoveryMaxBytesPerSec = INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.get(settings);
+        if (recoveryMaxBytesPerSec.getBytes() <= 0) {
+            recoveryRateLimiter = null;
         } else {
-            rateLimiter = new SimpleRateLimiter(maxBytesPerSec.getMbFrac());
+            recoveryRateLimiter = new SimpleRateLimiter(recoveryMaxBytesPerSec.getMbFrac());
         }
+        this.replicationMaxBytesPerSec = INDICES_REPLICATION_MAX_BYTES_PER_SEC_SETTING.get(settings);
+        updateReplicationRateLimiter();
 
-        logger.debug("using max_bytes_per_sec[{}]", maxBytesPerSec);
+        logger.debug("using recovery max_bytes_per_sec[{}]", recoveryMaxBytesPerSec);
         this.internalRemoteUploadTimeout = INDICES_INTERNAL_REMOTE_UPLOAD_TIMEOUT.get(settings);
 
-        clusterSettings.addSettingsUpdateConsumer(INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING, this::setMaxBytesPerSec);
+        clusterSettings.addSettingsUpdateConsumer(INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING, this::setRecoveryMaxBytesPerSec);
+        clusterSettings.addSettingsUpdateConsumer(INDICES_REPLICATION_MAX_BYTES_PER_SEC_SETTING, this::setReplicationMaxBytesPerSec);
         clusterSettings.addSettingsUpdateConsumer(INDICES_RECOVERY_MAX_CONCURRENT_FILE_CHUNKS_SETTING, this::setMaxConcurrentFileChunks);
         clusterSettings.addSettingsUpdateConsumer(INDICES_RECOVERY_MAX_CONCURRENT_OPERATIONS_SETTING, this::setMaxConcurrentOperations);
         clusterSettings.addSettingsUpdateConsumer(
@@ -243,17 +238,16 @@ public RecoverySettings(Settings settings, ClusterSettings clusterSettings) {
             this::setInternalActionLongTimeout
         );
         clusterSettings.addSettingsUpdateConsumer(INDICES_RECOVERY_ACTIVITY_TIMEOUT_SETTING, this::setActivityTimeout);
-        minRemoteSegmentMetadataFiles = CLUSTER_REMOTE_INDEX_SEGMENT_METADATA_RETENTION_MAX_COUNT_SETTING.get(settings);
-        clusterSettings.addSettingsUpdateConsumer(
-            CLUSTER_REMOTE_INDEX_SEGMENT_METADATA_RETENTION_MAX_COUNT_SETTING,
-            this::setMinRemoteSegmentMetadataFiles
-        );
         clusterSettings.addSettingsUpdateConsumer(INDICES_INTERNAL_REMOTE_UPLOAD_TIMEOUT, this::setInternalRemoteUploadTimeout);
 
     }
 
-    public RateLimiter rateLimiter() {
-        return rateLimiter;
+    public RateLimiter recoveryRateLimiter() {
+        return recoveryRateLimiter;
+    }
+
+    public RateLimiter replicationRateLimiter() {
+        return replicationRateLimiter;
     }
 
     public TimeValue retryDelayNetwork() {
@@ -319,14 +313,40 @@ public void setInternalRemoteUploadTimeout(TimeValue internalRemoteUploadTimeout
         this.internalRemoteUploadTimeout = internalRemoteUploadTimeout;
     }
 
-    private void setMaxBytesPerSec(ByteSizeValue maxBytesPerSec) {
-        this.maxBytesPerSec = maxBytesPerSec;
-        if (maxBytesPerSec.getBytes() <= 0) {
-            rateLimiter = null;
-        } else if (rateLimiter != null) {
-            rateLimiter.setMBPerSec(maxBytesPerSec.getMbFrac());
+    private void setRecoveryMaxBytesPerSec(ByteSizeValue recoveryMaxBytesPerSec) {
+        this.recoveryMaxBytesPerSec = recoveryMaxBytesPerSec;
+        if (recoveryMaxBytesPerSec.getBytes() <= 0) {
+            recoveryRateLimiter = null;
+        } else if (recoveryRateLimiter != null) {
+            recoveryRateLimiter.setMBPerSec(recoveryMaxBytesPerSec.getMbFrac());
         } else {
-            rateLimiter = new SimpleRateLimiter(maxBytesPerSec.getMbFrac());
+            recoveryRateLimiter = new SimpleRateLimiter(recoveryMaxBytesPerSec.getMbFrac());
+        }
+        if (replicationMaxBytesPerSec.getBytes() < 0) updateReplicationRateLimiter();
+    }
+
+    private void setReplicationMaxBytesPerSec(ByteSizeValue replicationMaxBytesPerSec) {
+        this.replicationMaxBytesPerSec = replicationMaxBytesPerSec;
+        updateReplicationRateLimiter();
+    }
+
+    private void updateReplicationRateLimiter() {
+        if (replicationMaxBytesPerSec.getBytes() >= 0) {
+            if (replicationMaxBytesPerSec.getBytes() == 0) {
+                replicationRateLimiter = null;
+            } else if (replicationRateLimiter != null) {
+                replicationRateLimiter.setMBPerSec(replicationMaxBytesPerSec.getMbFrac());
+            } else {
+                replicationRateLimiter = new SimpleRateLimiter(replicationMaxBytesPerSec.getMbFrac());
+            }
+        } else { // when replicationMaxBytesPerSec = -1B, use setting of recovery
+            if (recoveryMaxBytesPerSec.getBytes() <= 0) {
+                replicationRateLimiter = null;
+            } else if (replicationRateLimiter != null) {
+                replicationRateLimiter.setMBPerSec(recoveryMaxBytesPerSec.getMbFrac());
+            } else {
+                replicationRateLimiter = new SimpleRateLimiter(recoveryMaxBytesPerSec.getMbFrac());
+            }
         }
     }
 
@@ -354,11 +374,4 @@ private void setMaxConcurrentRemoteStoreStreams(int maxConcurrentRemoteStoreStre
         this.maxConcurrentRemoteStoreStreams = maxConcurrentRemoteStoreStreams;
     }
 
-    private void setMinRemoteSegmentMetadataFiles(int minRemoteSegmentMetadataFiles) {
-        this.minRemoteSegmentMetadataFiles = minRemoteSegmentMetadataFiles;
-    }
-
-    public int getMinRemoteSegmentMetadataFiles() {
-        return this.minRemoteSegmentMetadataFiles;
-    }
 }
diff --git a/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandlerFactory.java b/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandlerFactory.java
index 0ccb1ac2133cf..96e85154e6248 100644
--- a/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandlerFactory.java
+++ b/server/src/main/java/org/opensearch/indices/recovery/RecoverySourceHandlerFactory.java
@@ -23,8 +23,7 @@ public static RecoverySourceHandler create(
         StartRecoveryRequest request,
         RecoverySettings recoverySettings
     ) {
-        boolean isReplicaRecoveryWithRemoteTranslog = request.isPrimaryRelocation() == false
-            && (shard.isRemoteTranslogEnabled() || shard.isMigratingToRemote());
+        boolean isReplicaRecoveryWithRemoteTranslog = request.isPrimaryRelocation() == false && request.targetNode().isRemoteStoreNode();
         if (isReplicaRecoveryWithRemoteTranslog) {
             return new RemoteStorePeerRecoverySourceHandler(
                 shard,
diff --git a/server/src/main/java/org/opensearch/indices/recovery/RecoveryTarget.java b/server/src/main/java/org/opensearch/indices/recovery/RecoveryTarget.java
index 16311d5d2cfb7..f47b082de3856 100644
--- a/server/src/main/java/org/opensearch/indices/recovery/RecoveryTarget.java
+++ b/server/src/main/java/org/opensearch/indices/recovery/RecoveryTarget.java
@@ -213,10 +213,17 @@ public void prepareForTranslogOperations(int totalTranslogOps, ActionListener<Vo
         ActionListener.completeWith(listener, () -> {
             state().getIndex().setFileDetailsComplete(); // ops-based recoveries don't send the file details
             state().getTranslog().totalOperations(totalTranslogOps);
+            // Cleanup remote contents before opening new translog.
+            // This prevents reading from any old Translog UUIDs during re-seeding
+            // (situation in which primary fails over to docrep replica and is re-seeded to remote again)
+            // which might end up causing a TranslogCorruptedException
+            if (indexShard.shouldSeedRemoteStore()) {
+                assert indexShard.routingEntry().primary() : "Remote seeding should only true be for primary shard copy";
+                indexShard.deleteRemoteStoreContents();
+            }
             indexShard().openEngineAndSkipTranslogRecovery();
             // upload to remote store in migration for primary shard
-            if (indexShard.shouldSeedRemoteStore() && indexShard.routingEntry().primary()) {
-                indexShard.deleteRemoteStoreContents();
+            if (indexShard.shouldSeedRemoteStore()) {
                 // This cleans up remote translog's 0 generation, as we don't want to get that uploaded
                 indexShard.sync();
                 threadPool.executor(ThreadPool.Names.GENERIC).execute(() -> { indexShard.refresh("remote store migration"); });
diff --git a/server/src/main/java/org/opensearch/indices/recovery/RemoteRecoveryTargetHandler.java b/server/src/main/java/org/opensearch/indices/recovery/RemoteRecoveryTargetHandler.java
index 37227596fdfe7..4d4c20f778ef3 100644
--- a/server/src/main/java/org/opensearch/indices/recovery/RemoteRecoveryTargetHandler.java
+++ b/server/src/main/java/org/opensearch/indices/recovery/RemoteRecoveryTargetHandler.java
@@ -111,7 +111,8 @@ public RemoteRecoveryTargetHandler(
             shardId,
             PeerRecoveryTargetService.Actions.FILE_CHUNK,
             requestSeqNoGenerator,
-            onSourceThrottle
+            onSourceThrottle,
+            recoverySettings::recoveryRateLimiter
         );
         this.remoteStoreEnabled = remoteStoreEnabled;
     }
diff --git a/server/src/main/java/org/opensearch/indices/replication/CheckpointInfoResponse.java b/server/src/main/java/org/opensearch/indices/replication/CheckpointInfoResponse.java
index 9fd3b7f3afb80..24b744bebc53d 100644
--- a/server/src/main/java/org/opensearch/indices/replication/CheckpointInfoResponse.java
+++ b/server/src/main/java/org/opensearch/indices/replication/CheckpointInfoResponse.java
@@ -40,6 +40,12 @@ public CheckpointInfoResponse(
         this.infosBytes = infosBytes;
     }
 
+    public CheckpointInfoResponse(final ReplicationCheckpoint checkpoint, final byte[] infosBytes) {
+        this.checkpoint = checkpoint;
+        this.infosBytes = infosBytes;
+        this.metadataMap = checkpoint.getMetadataMap();
+    }
+
     public CheckpointInfoResponse(StreamInput in) throws IOException {
         this.checkpoint = new ReplicationCheckpoint(in);
         this.metadataMap = in.readMap(StreamInput::readString, StoreFileMetadata::new);
diff --git a/server/src/main/java/org/opensearch/indices/replication/OngoingSegmentReplications.java b/server/src/main/java/org/opensearch/indices/replication/OngoingSegmentReplications.java
index 33967c0203516..6b99b3c0b0696 100644
--- a/server/src/main/java/org/opensearch/indices/replication/OngoingSegmentReplications.java
+++ b/server/src/main/java/org/opensearch/indices/replication/OngoingSegmentReplications.java
@@ -21,12 +21,10 @@
 import org.opensearch.indices.IndicesService;
 import org.opensearch.indices.recovery.FileChunkWriter;
 import org.opensearch.indices.recovery.RecoverySettings;
-import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint;
-import org.opensearch.indices.replication.common.CopyState;
 
 import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -36,7 +34,6 @@
 /**
  * Manages references to ongoing segrep events on a node.
  * Each replica will have a new {@link SegmentReplicationSourceHandler} created when starting replication.
- * CopyStates will be cached for reuse between replicas and only released when all replicas have finished copying segments.
  *
  * @opensearch.internal
  */
@@ -45,7 +42,6 @@ class OngoingSegmentReplications {
     private static final Logger logger = LogManager.getLogger(OngoingSegmentReplications.class);
     private final RecoverySettings recoverySettings;
     private final IndicesService indicesService;
-    private final Map<ReplicationCheckpoint, CopyState> copyStateMap;
     private final Map<String, SegmentReplicationSourceHandler> allocationIdToHandlers;
 
     /**
@@ -57,46 +53,9 @@ class OngoingSegmentReplications {
     OngoingSegmentReplications(IndicesService indicesService, RecoverySettings recoverySettings) {
         this.indicesService = indicesService;
         this.recoverySettings = recoverySettings;
-        this.copyStateMap = Collections.synchronizedMap(new HashMap<>());
         this.allocationIdToHandlers = ConcurrentCollections.newConcurrentMap();
     }
 
-    /*
-      Operations on the {@link #copyStateMap} member.
-     */
-
-    /**
-     * A synchronized method that checks {@link #copyStateMap} for the given {@link ReplicationCheckpoint} key
-     * and returns the cached value if one is present. If the key is not present, a {@link CopyState}
-     * object is constructed and stored in the map before being returned.
-     */
-    synchronized CopyState getCachedCopyState(ReplicationCheckpoint checkpoint) throws IOException {
-        if (isInCopyStateMap(checkpoint)) {
-            final CopyState copyState = fetchFromCopyStateMap(checkpoint);
-            // we incref the copyState for every replica that is using this checkpoint.
-            // decref will happen when copy completes.
-            copyState.incRef();
-            return copyState;
-        } else {
-            // From the checkpoint's shard ID, fetch the IndexShard
-            ShardId shardId = checkpoint.getShardId();
-            final IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
-            final IndexShard indexShard = indexService.getShard(shardId.id());
-            // build the CopyState object and cache it before returning
-            final CopyState copyState = new CopyState(checkpoint, indexShard);
-
-            /*
-              Use the checkpoint from the request as the key in the map, rather than
-              the checkpoint from the created CopyState. This maximizes cache hits
-              if replication targets make a request with an older checkpoint.
-              Replication targets are expected to fetch the checkpoint in the response
-              CopyState to bring themselves up to date.
-             */
-            addToCopyStateMap(checkpoint, copyState);
-            return copyState;
-        }
-    }
-
     /**
      * Start sending files to the replica.
      *
@@ -114,12 +73,10 @@ void startSegmentCopy(GetSegmentFilesRequest request, ActionListener<GetSegmentF
                 );
             }
             // update the given listener to release the CopyState before it resolves.
-            final ActionListener<GetSegmentFilesResponse> wrappedListener = ActionListener.runBefore(listener, () -> {
-                final SegmentReplicationSourceHandler sourceHandler = allocationIdToHandlers.remove(request.getTargetAllocationId());
-                if (sourceHandler != null) {
-                    removeCopyState(sourceHandler.getCopyState());
-                }
-            });
+            final ActionListener<GetSegmentFilesResponse> wrappedListener = ActionListener.runBefore(
+                listener,
+                () -> allocationIdToHandlers.remove(request.getTargetAllocationId())
+            );
             handler.sendFiles(request, wrappedListener);
         } else {
             listener.onResponse(new GetSegmentFilesResponse(Collections.emptyList()));
@@ -127,38 +84,32 @@ void startSegmentCopy(GetSegmentFilesRequest request, ActionListener<GetSegmentF
     }
 
     /**
-     * Prepare for a Replication event. This method constructs a {@link CopyState} holding files to be sent off of the current
-     * node's store.  This state is intended to be sent back to Replicas before copy is initiated so the replica can perform a diff against its
-     * local store.  It will then build a handler to orchestrate the segment copy that will be stored locally and started on a subsequent request from replicas
-     * with the list of required files.
+     * Prepare for a Replication event. This method constructs a {@link SegmentReplicationSourceHandler} that orchestrates segment copy and
+     * will internally incref files for copy.
      *
      * @param request         {@link CheckpointInfoRequest}
      * @param fileChunkWriter {@link FileChunkWriter} writer to handle sending files over the transport layer.
-     * @return {@link CopyState} the built CopyState for this replication event.
-     * @throws IOException - When there is an IO error building CopyState.
+     * @return {@link SegmentReplicationSourceHandler} the built CopyState for this replication event.
      */
-    CopyState prepareForReplication(CheckpointInfoRequest request, FileChunkWriter fileChunkWriter) throws IOException {
-        final CopyState copyState = getCachedCopyState(request.getCheckpoint());
-        final SegmentReplicationSourceHandler newHandler = createTargetHandler(
-            request.getTargetNode(),
-            copyState,
-            request.getTargetAllocationId(),
-            fileChunkWriter
-        );
-        final SegmentReplicationSourceHandler existingHandler = allocationIdToHandlers.putIfAbsent(
-            request.getTargetAllocationId(),
-            newHandler
-        );
-        // If we are already replicating to this allocation Id, cancel the old and replace with a new execution.
-        // This will clear the old handler & referenced copy state holding an incref'd indexCommit.
-        if (existingHandler != null) {
-            logger.warn("Override handler for allocation id {}", request.getTargetAllocationId());
-            cancelHandlers(handler -> handler.getAllocationId().equals(request.getTargetAllocationId()), "cancel due to retry");
-            assert allocationIdToHandlers.containsKey(request.getTargetAllocationId()) == false;
-            allocationIdToHandlers.put(request.getTargetAllocationId(), newHandler);
-        }
-        assert allocationIdToHandlers.containsKey(request.getTargetAllocationId());
-        return copyState;
+    SegmentReplicationSourceHandler prepareForReplication(CheckpointInfoRequest request, FileChunkWriter fileChunkWriter) {
+        return allocationIdToHandlers.computeIfAbsent(request.getTargetAllocationId(), aId -> {
+            try {
+                // From the checkpoint's shard ID, fetch the IndexShard
+                final ShardId shardId = request.getCheckpoint().getShardId();
+                final IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex());
+                final IndexShard indexShard = indexService.getShard(shardId.id());
+                return new SegmentReplicationSourceHandler(
+                    request.getTargetNode(),
+                    fileChunkWriter,
+                    indexShard,
+                    request.getTargetAllocationId(),
+                    Math.toIntExact(recoverySettings.getChunkSize().getBytes()),
+                    recoverySettings.getMaxConcurrentFileChunks()
+                );
+            } catch (IOException e) {
+                throw new UncheckedIOException("Error creating replication handler", e);
+            }
+        });
     }
 
     /**
@@ -167,8 +118,8 @@ CopyState prepareForReplication(CheckpointInfoRequest request, FileChunkWriter f
      * @param shard  {@link IndexShard}
      * @param reason {@link String} - Reason for the cancel
      */
-    synchronized void cancel(IndexShard shard, String reason) {
-        cancelHandlers(handler -> handler.getCopyState().getShard().shardId().equals(shard.shardId()), reason);
+    void cancel(IndexShard shard, String reason) {
+        cancelHandlers(handler -> handler.shardId().equals(shard.shardId()), reason);
     }
 
     /**
@@ -177,11 +128,10 @@ synchronized void cancel(IndexShard shard, String reason) {
      * @param allocationId {@link String} - Allocation ID.
      * @param reason       {@link String} - Reason for the cancel
      */
-    synchronized void cancel(String allocationId, String reason) {
+    void cancel(String allocationId, String reason) {
         final SegmentReplicationSourceHandler handler = allocationIdToHandlers.remove(allocationId);
         if (handler != null) {
             handler.cancel(reason);
-            removeCopyState(handler.getCopyState());
         }
     }
 
@@ -194,14 +144,6 @@ void cancelReplication(DiscoveryNode node) {
         cancelHandlers(handler -> handler.getTargetNode().equals(node), "Node left");
     }
 
-    /**
-     * Checks if the {@link #copyStateMap} has the input {@link ReplicationCheckpoint}
-     * as a key by invoking {@link Map#containsKey(Object)}.
-     */
-    boolean isInCopyStateMap(ReplicationCheckpoint replicationCheckpoint) {
-        return copyStateMap.containsKey(replicationCheckpoint);
-    }
-
     int size() {
         return allocationIdToHandlers.size();
     }
@@ -211,58 +153,20 @@ Map<String, SegmentReplicationSourceHandler> getHandlers() {
         return allocationIdToHandlers;
     }
 
-    int cachedCopyStateSize() {
-        return copyStateMap.size();
-    }
-
-    private SegmentReplicationSourceHandler createTargetHandler(
-        DiscoveryNode node,
-        CopyState copyState,
-        String allocationId,
-        FileChunkWriter fileChunkWriter
-    ) {
-        return new SegmentReplicationSourceHandler(
-            node,
-            fileChunkWriter,
-            copyState.getShard().getThreadPool(),
-            copyState,
-            allocationId,
-            Math.toIntExact(recoverySettings.getChunkSize().getBytes()),
-            recoverySettings.getMaxConcurrentFileChunks()
-        );
-    }
-
     /**
-     * Adds the input {@link CopyState} object to {@link #copyStateMap}.
-     * The key is the CopyState's {@link ReplicationCheckpoint} object.
-     */
-    private void addToCopyStateMap(ReplicationCheckpoint checkpoint, CopyState copyState) {
-        copyStateMap.putIfAbsent(checkpoint, copyState);
-    }
-
-    /**
-     * Given a {@link ReplicationCheckpoint}, return the corresponding
-     * {@link CopyState} object, if any, from {@link #copyStateMap}.
-     */
-    private CopyState fetchFromCopyStateMap(ReplicationCheckpoint replicationCheckpoint) {
-        return copyStateMap.get(replicationCheckpoint);
-    }
-
-    /**
-     * Remove a CopyState. Intended to be called after a replication event completes.
-     * This method will remove a copyState from the copyStateMap only if its refCount hits 0.
-     *
-     * @param copyState {@link CopyState}
+     * Clear handlers for any allocationIds not in sync.
+     * @param shardId {@link ShardId}
+     * @param inSyncAllocationIds {@link List} of in-sync allocation Ids.
      */
-    private synchronized void removeCopyState(CopyState copyState) {
-        if (copyState.decRef() == true) {
-            copyStateMap.remove(copyState.getRequestedReplicationCheckpoint());
-        }
+    void clearOutOfSyncIds(ShardId shardId, Set<String> inSyncAllocationIds) {
+        cancelHandlers(
+            (handler) -> handler.shardId().equals(shardId) && inSyncAllocationIds.contains(handler.getAllocationId()) == false,
+            "Shard is no longer in-sync with the primary"
+        );
     }
 
     /**
      * Remove handlers from allocationIdToHandlers map based on a filter predicate.
-     * This will also decref the handler's CopyState reference.
      */
     private void cancelHandlers(Predicate<? super SegmentReplicationSourceHandler> predicate, String reason) {
         final List<String> allocationIds = allocationIdToHandlers.values()
@@ -278,17 +182,4 @@ private void cancelHandlers(Predicate<? super SegmentReplicationSourceHandler> p
             cancel(allocationId, reason);
         }
     }
-
-    /**
-     * Clear copystate and target handlers for any non insync allocationIds.
-     * @param shardId {@link ShardId}
-     * @param inSyncAllocationIds {@link List} of in-sync allocation Ids.
-     */
-    public void clearOutOfSyncIds(ShardId shardId, Set<String> inSyncAllocationIds) {
-        cancelHandlers(
-            (handler) -> handler.getCopyState().getShard().shardId().equals(shardId)
-                && inSyncAllocationIds.contains(handler.getAllocationId()) == false,
-            "Shard is no longer in-sync with the primary"
-        );
-    }
 }
diff --git a/server/src/main/java/org/opensearch/indices/replication/RemoteSegmentFileChunkWriter.java b/server/src/main/java/org/opensearch/indices/replication/RemoteSegmentFileChunkWriter.java
index b52fe66816098..179e497565326 100644
--- a/server/src/main/java/org/opensearch/indices/replication/RemoteSegmentFileChunkWriter.java
+++ b/server/src/main/java/org/opensearch/indices/replication/RemoteSegmentFileChunkWriter.java
@@ -25,6 +25,7 @@
 import java.io.IOException;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.Consumer;
+import java.util.function.Supplier;
 
 /**
  * This class handles sending file chunks over the transport layer to a target shard.
@@ -36,11 +37,11 @@ public final class RemoteSegmentFileChunkWriter implements FileChunkWriter {
     private final AtomicLong requestSeqNoGenerator;
     private final RetryableTransportClient retryableTransportClient;
     private final ShardId shardId;
-    private final RecoverySettings recoverySettings;
     private final long replicationId;
     private final AtomicLong bytesSinceLastPause = new AtomicLong();
     private final TransportRequestOptions fileChunkRequestOptions;
     private final Consumer<Long> onSourceThrottle;
+    private final Supplier<RateLimiter> rateLimiterSupplier;
     private final String action;
 
     public RemoteSegmentFileChunkWriter(
@@ -50,14 +51,15 @@ public RemoteSegmentFileChunkWriter(
         ShardId shardId,
         String action,
         AtomicLong requestSeqNoGenerator,
-        Consumer<Long> onSourceThrottle
+        Consumer<Long> onSourceThrottle,
+        Supplier<RateLimiter> rateLimiterSupplier
     ) {
         this.replicationId = replicationId;
-        this.recoverySettings = recoverySettings;
         this.retryableTransportClient = retryableTransportClient;
         this.shardId = shardId;
         this.requestSeqNoGenerator = requestSeqNoGenerator;
         this.onSourceThrottle = onSourceThrottle;
+        this.rateLimiterSupplier = rateLimiterSupplier;
         this.fileChunkRequestOptions = TransportRequestOptions.builder()
             .withType(TransportRequestOptions.Type.RECOVERY)
             .withTimeout(recoverySettings.internalActionTimeout())
@@ -78,7 +80,7 @@ public void writeFileChunk(
         // Pause using the rate limiter, if desired, to throttle the recovery
         final long throttleTimeInNanos;
         // always fetch the ratelimiter - it might be updated in real-time on the recovery settings
-        final RateLimiter rl = recoverySettings.rateLimiter();
+        final RateLimiter rl = rateLimiterSupplier.get();
         if (rl != null) {
             long bytes = bytesSinceLastPause.addAndGet(content.length());
             if (bytes > rl.getMinPauseCheckBytes()) {
diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceFactory.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceFactory.java
index 852003c9f3e4d..81eb38757aebe 100644
--- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceFactory.java
+++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceFactory.java
@@ -38,7 +38,7 @@ public SegmentReplicationSourceFactory(
     }
 
     public SegmentReplicationSource get(IndexShard shard) {
-        if (shard.indexSettings().isSegRepWithRemoteEnabled()) {
+        if (shard.indexSettings().isAssignedOnRemoteNode()) {
             return new RemoteStoreReplicationSource(shard);
         } else {
             return new PrimaryShardReplicationSource(
diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceHandler.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceHandler.java
index 674c09311c645..bb64d6b0c60b6 100644
--- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceHandler.java
+++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceHandler.java
@@ -18,16 +18,18 @@
 import org.opensearch.common.util.concurrent.OpenSearchExecutors;
 import org.opensearch.common.util.io.IOUtils;
 import org.opensearch.core.action.ActionListener;
+import org.opensearch.core.index.shard.ShardId;
 import org.opensearch.index.shard.IndexShard;
 import org.opensearch.index.store.StoreFileMetadata;
 import org.opensearch.indices.recovery.FileChunkWriter;
 import org.opensearch.indices.recovery.MultiChunkTransfer;
+import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint;
 import org.opensearch.indices.replication.common.CopyState;
 import org.opensearch.indices.replication.common.ReplicationTimer;
-import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.transport.Transports;
 
 import java.io.Closeable;
+import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashSet;
@@ -54,48 +56,46 @@ class SegmentReplicationSourceHandler {
     private final AtomicBoolean isReplicating = new AtomicBoolean();
     private final DiscoveryNode targetNode;
     private final String allocationId;
-
     private final FileChunkWriter writer;
 
     /**
      * Constructor.
      *
-     * @param targetNode              - {@link DiscoveryNode} target node where files should be sent.
+     * @param targetNode              {@link DiscoveryNode} target node where files should be sent.
      * @param writer                  {@link FileChunkWriter} implementation that sends file chunks over the transport layer.
-     * @param threadPool              {@link ThreadPool} Thread pool.
-     * @param copyState               {@link CopyState} CopyState holding segment file metadata.
+     * @param shard                   {@link IndexShard} The primary shard local to this node.
      * @param fileChunkSizeInBytes    {@link Integer}
      * @param maxConcurrentFileChunks {@link Integer}
      */
     SegmentReplicationSourceHandler(
         DiscoveryNode targetNode,
         FileChunkWriter writer,
-        ThreadPool threadPool,
-        CopyState copyState,
+        IndexShard shard,
         String allocationId,
         int fileChunkSizeInBytes,
         int maxConcurrentFileChunks
-    ) {
+    ) throws IOException {
         this.targetNode = targetNode;
-        this.shard = copyState.getShard();
+        this.shard = shard;
         this.logger = Loggers.getLogger(
             SegmentReplicationSourceHandler.class,
-            copyState.getShard().shardId(),
+            shard.shardId(),
             "sending segments to " + targetNode.getName()
         );
         this.segmentFileTransferHandler = new SegmentFileTransferHandler(
-            copyState.getShard(),
+            shard,
             targetNode,
             writer,
             logger,
-            threadPool,
+            shard.getThreadPool(),
             cancellableThreads,
             fileChunkSizeInBytes,
             maxConcurrentFileChunks
         );
         this.allocationId = allocationId;
-        this.copyState = copyState;
+        this.copyState = new CopyState(shard);
         this.writer = writer;
+        resources.add(copyState);
     }
 
     /**
@@ -109,6 +109,7 @@ public synchronized void sendFiles(GetSegmentFilesRequest request, ActionListene
         if (request.getFilesToFetch().isEmpty()) {
             // before completion, alert the primary of the replica's state.
             shard.updateVisibleCheckpointForShard(request.getTargetAllocationId(), copyState.getCheckpoint());
+            IOUtils.closeWhileHandlingException(copyState);
             listener.onResponse(new GetSegmentFilesResponse(Collections.emptyList()));
             return;
         }
@@ -183,10 +184,7 @@ public synchronized void sendFiles(GetSegmentFilesRequest request, ActionListene
     public void cancel(String reason) {
         writer.cancel();
         cancellableThreads.cancel(reason);
-    }
-
-    CopyState getCopyState() {
-        return copyState;
+        IOUtils.closeWhileHandlingException(copyState);
     }
 
     public boolean isReplicating() {
@@ -200,4 +198,16 @@ public DiscoveryNode getTargetNode() {
     public String getAllocationId() {
         return allocationId;
     }
+
+    public ReplicationCheckpoint getCheckpoint() {
+        return copyState.getCheckpoint();
+    }
+
+    public byte[] getInfosBytes() {
+        return copyState.getInfosBytes();
+    }
+
+    public ShardId shardId() {
+        return shard.shardId();
+    }
 }
diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceService.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceService.java
index a393faabae0ea..21fd066b8be2f 100644
--- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceService.java
+++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationSourceService.java
@@ -29,7 +29,6 @@
 import org.opensearch.indices.IndicesService;
 import org.opensearch.indices.recovery.RecoverySettings;
 import org.opensearch.indices.recovery.RetryableTransportClient;
-import org.opensearch.indices.replication.common.CopyState;
 import org.opensearch.indices.replication.common.ReplicationTimer;
 import org.opensearch.tasks.Task;
 import org.opensearch.threadpool.ThreadPool;
@@ -124,18 +123,20 @@ public void messageReceived(CheckpointInfoRequest request, TransportChannel chan
                 request.getCheckpoint().getShardId(),
                 SegmentReplicationTargetService.Actions.FILE_CHUNK,
                 new AtomicLong(0),
-                (throttleTime) -> {}
+                (throttleTime) -> {},
+                recoverySettings::replicationRateLimiter
             );
-            final CopyState copyState = ongoingSegmentReplications.prepareForReplication(request, segmentSegmentFileChunkWriter);
-            channel.sendResponse(
-                new CheckpointInfoResponse(copyState.getCheckpoint(), copyState.getMetadataMap(), copyState.getInfosBytes())
+            final SegmentReplicationSourceHandler handler = ongoingSegmentReplications.prepareForReplication(
+                request,
+                segmentSegmentFileChunkWriter
             );
+            channel.sendResponse(new CheckpointInfoResponse(handler.getCheckpoint(), handler.getInfosBytes()));
             timer.stop();
             logger.trace(
                 new ParameterizedMessage(
                     "[replication id {}] Source node sent checkpoint info [{}] to target node [{}], timing: {}",
                     request.getReplicationId(),
-                    copyState.getCheckpoint(),
+                    handler.getCheckpoint(),
                     request.getTargetNode().getId(),
                     timer.time()
                 )
@@ -217,7 +218,7 @@ protected void doClose() throws IOException {
 
     /**
      *
-     * Cancels any replications on this node to a replica shard that is about to be closed.
+     * Before a primary shard is closed, cancel any ongoing replications to release incref'd segments.
      */
     @Override
     public void beforeIndexShardClosed(ShardId shardId, @Nullable IndexShard indexShard, Settings indexSettings) {
diff --git a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java
index 4942d39cfa48a..fbd7ab7cea346 100644
--- a/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java
+++ b/server/src/main/java/org/opensearch/indices/replication/SegmentReplicationTargetService.java
@@ -635,7 +635,7 @@ public void messageReceived(final FileChunkRequest request, TransportChannel cha
             try (ReplicationRef<SegmentReplicationTarget> ref = onGoingReplications.getSafe(request.recoveryId(), request.shardId())) {
                 final SegmentReplicationTarget target = ref.get();
                 final ActionListener<Void> listener = target.createOrFinishListener(channel, Actions.FILE_CHUNK, request);
-                target.handleFileChunk(request, target, bytesSinceLastPause, recoverySettings.rateLimiter(), listener);
+                target.handleFileChunk(request, target, bytesSinceLastPause, recoverySettings.replicationRateLimiter(), listener);
             }
         }
     }
diff --git a/server/src/main/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointAction.java b/server/src/main/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointAction.java
index 821ae42e31881..8f39aa194b06c 100644
--- a/server/src/main/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointAction.java
+++ b/server/src/main/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointAction.java
@@ -98,7 +98,7 @@ protected void doExecute(Task task, PublishCheckpointRequest request, ActionList
 
     @Override
     public ReplicationMode getReplicationMode(IndexShard indexShard) {
-        if (indexShard.isRemoteTranslogEnabled()) {
+        if (indexShard.indexSettings().isAssignedOnRemoteNode()) {
             return ReplicationMode.FULL_REPLICATION;
         }
         return super.getReplicationMode(indexShard);
@@ -199,6 +199,12 @@ protected void shardOperationOnReplica(PublishCheckpointRequest request, IndexSh
         Objects.requireNonNull(replica);
         ActionListener.completeWith(listener, () -> {
             logger.trace(() -> new ParameterizedMessage("Checkpoint {} received on replica {}", request, replica.shardId()));
+            // Condition for ensuring that we ignore Segrep checkpoints received on Docrep shard copies.
+            // This case will hit iff the replica hosting node is not remote enabled and replication type != SEGMENT
+            if (replica.indexSettings().isAssignedOnRemoteNode() == false && replica.indexSettings().isSegRepLocalEnabled() == false) {
+                logger.trace("Received segrep checkpoint on a docrep shard copy during an ongoing remote migration. NoOp.");
+                return new ReplicaResult();
+            }
             if (request.getCheckpoint().getShardId().equals(replica.shardId())) {
                 replicationService.onNewCheckpoint(request.getCheckpoint(), replica);
             }
diff --git a/server/src/main/java/org/opensearch/indices/replication/common/CopyState.java b/server/src/main/java/org/opensearch/indices/replication/common/CopyState.java
index 3b7ae2af80ca0..7d3eb9083208b 100644
--- a/server/src/main/java/org/opensearch/indices/replication/common/CopyState.java
+++ b/server/src/main/java/org/opensearch/indices/replication/common/CopyState.java
@@ -13,11 +13,11 @@
 import org.apache.lucene.store.ByteBuffersIndexOutput;
 import org.opensearch.common.collect.Tuple;
 import org.opensearch.common.concurrent.GatedCloseable;
-import org.opensearch.common.util.concurrent.AbstractRefCounted;
 import org.opensearch.index.shard.IndexShard;
 import org.opensearch.index.store.StoreFileMetadata;
 import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint;
 
+import java.io.Closeable;
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.util.Map;
@@ -28,28 +28,21 @@
  *
  * @opensearch.internal
  */
-public class CopyState extends AbstractRefCounted {
+public class CopyState implements Closeable {
 
     private final GatedCloseable<SegmentInfos> segmentInfosRef;
-    /** ReplicationCheckpoint requested */
-    private final ReplicationCheckpoint requestedReplicationCheckpoint;
     /** Actual ReplicationCheckpoint returned by the shard */
     private final ReplicationCheckpoint replicationCheckpoint;
-    private final Map<String, StoreFileMetadata> metadataMap;
     private final byte[] infosBytes;
     private final IndexShard shard;
 
-    public CopyState(ReplicationCheckpoint requestedReplicationCheckpoint, IndexShard shard) throws IOException {
-        super("CopyState-" + shard.shardId());
-        this.requestedReplicationCheckpoint = requestedReplicationCheckpoint;
+    public CopyState(IndexShard shard) throws IOException {
         this.shard = shard;
         final Tuple<GatedCloseable<SegmentInfos>, ReplicationCheckpoint> latestSegmentInfosAndCheckpoint = shard
             .getLatestSegmentInfosAndCheckpoint();
         this.segmentInfosRef = latestSegmentInfosAndCheckpoint.v1();
         this.replicationCheckpoint = latestSegmentInfosAndCheckpoint.v2();
         SegmentInfos segmentInfos = this.segmentInfosRef.get();
-        this.metadataMap = shard.store().getSegmentMetadataMap(segmentInfos);
-
         ByteBuffersDataOutput buffer = new ByteBuffersDataOutput();
         // resource description and name are not used, but resource description cannot be null
         try (ByteBuffersIndexOutput indexOutput = new ByteBuffersIndexOutput(buffer, "", null)) {
@@ -58,21 +51,12 @@ public CopyState(ReplicationCheckpoint requestedReplicationCheckpoint, IndexShar
         this.infosBytes = buffer.toArrayCopy();
     }
 
-    @Override
-    protected void closeInternal() {
-        try {
-            segmentInfosRef.close();
-        } catch (IOException e) {
-            throw new UncheckedIOException(e);
-        }
-    }
-
     public ReplicationCheckpoint getCheckpoint() {
         return replicationCheckpoint;
     }
 
     public Map<String, StoreFileMetadata> getMetadataMap() {
-        return metadataMap;
+        return replicationCheckpoint.getMetadataMap();
     }
 
     public byte[] getInfosBytes() {
@@ -83,7 +67,12 @@ public IndexShard getShard() {
         return shard;
     }
 
-    public ReplicationCheckpoint getRequestedReplicationCheckpoint() {
-        return requestedReplicationCheckpoint;
+    @Override
+    public void close() throws IOException {
+        try {
+            segmentInfosRef.close();
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
     }
 }
diff --git a/server/src/main/java/org/opensearch/indices/store/ShardAttributes.java b/server/src/main/java/org/opensearch/indices/store/ShardAttributes.java
index 4ef4e91f7af8c..999acd02030ab 100644
--- a/server/src/main/java/org/opensearch/indices/store/ShardAttributes.java
+++ b/server/src/main/java/org/opensearch/indices/store/ShardAttributes.java
@@ -12,36 +12,27 @@
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
-import org.opensearch.core.index.shard.ShardId;
-import org.opensearch.gateway.AsyncShardFetch;
 
 import java.io.IOException;
 
 /**
- * This class contains information about the shard that needs to be sent as part of request in Transport Action implementing
- * {@link AsyncShardFetch.Lister} to fetch shard information in async manner
+ * This class contains Attributes related to Shards that are necessary for making the
+ * {@link org.opensearch.gateway.TransportNodesListGatewayStartedShards} transport requests
  *
  * @opensearch.internal
  */
 public class ShardAttributes implements Writeable {
-    private final ShardId shardId;
     @Nullable
     private final String customDataPath;
 
-    public ShardAttributes(ShardId shardId, String customDataPath) {
-        this.shardId = shardId;
+    public ShardAttributes(String customDataPath) {
         this.customDataPath = customDataPath;
     }
 
     public ShardAttributes(StreamInput in) throws IOException {
-        shardId = new ShardId(in);
         customDataPath = in.readString();
     }
 
-    public ShardId getShardId() {
-        return shardId;
-    }
-
     /**
      * Returns the custom data path that is used to look up information for this shard.
      * Returns an empty string if no custom data path is used for this index.
@@ -53,7 +44,11 @@ public String getCustomDataPath() {
     }
 
     public void writeTo(StreamOutput out) throws IOException {
-        shardId.writeTo(out);
         out.writeString(customDataPath);
     }
+
+    @Override
+    public String toString() {
+        return "ShardAttributes{" + ", customDataPath='" + customDataPath + '\'' + '}';
+    }
 }
diff --git a/server/src/main/java/org/opensearch/indices/store/TransportNodesListShardStoreMetadataBatch.java b/server/src/main/java/org/opensearch/indices/store/TransportNodesListShardStoreMetadataBatch.java
index 3f151fe1c5ca0..22b03539cca74 100644
--- a/server/src/main/java/org/opensearch/indices/store/TransportNodesListShardStoreMetadataBatch.java
+++ b/server/src/main/java/org/opensearch/indices/store/TransportNodesListShardStoreMetadataBatch.java
@@ -155,7 +155,7 @@ private Map<ShardId, NodeStoreFilesMetadata> listStoreMetadata(NodeRequest reque
                 shardStoreMetadataMap.put(shardId, new NodeStoreFilesMetadata(storeFilesMetadata, null));
             } catch (Exception e) {
                 // should return null in case of known exceptions being returned from listShardMetadataInternal method.
-                if (e.getMessage().contains(INDEX_NOT_FOUND)) {
+                if (e.getMessage().contains(INDEX_NOT_FOUND) || e instanceof IOException) {
                     shardStoreMetadataMap.put(shardId, null);
                 } else {
                     // return actual exception as it is for unknown exceptions
@@ -276,6 +276,11 @@ public void writeTo(StreamOutput out) throws IOException {
             }
         }
 
+        public static boolean isEmpty(NodeStoreFilesMetadata response) {
+            return response.storeFilesMetadata() == null
+                || response.storeFilesMetadata().isEmpty() && response.getStoreFileFetchException() == null;
+        }
+
         public Exception getStoreFileFetchException() {
             return storeFileFetchException;
         }
@@ -324,7 +329,13 @@ public static class NodeStoreFilesMetadataBatch extends BaseNodeResponse {
 
         protected NodeStoreFilesMetadataBatch(StreamInput in) throws IOException {
             super(in);
-            this.nodeStoreFilesMetadataBatch = in.readMap(ShardId::new, NodeStoreFilesMetadata::new);
+            this.nodeStoreFilesMetadataBatch = in.readMap(ShardId::new, i -> {
+                if (i.readBoolean()) {
+                    return new NodeStoreFilesMetadata(i);
+                } else {
+                    return null;
+                }
+            });
         }
 
         public NodeStoreFilesMetadataBatch(DiscoveryNode node, Map<ShardId, NodeStoreFilesMetadata> nodeStoreFilesMetadataBatch) {
@@ -339,7 +350,14 @@ public Map<ShardId, NodeStoreFilesMetadata> getNodeStoreFilesMetadataBatch() {
         @Override
         public void writeTo(StreamOutput out) throws IOException {
             super.writeTo(out);
-            out.writeMap(nodeStoreFilesMetadataBatch, (o, k) -> k.writeTo(o), (o, v) -> v.writeTo(o));
+            out.writeMap(nodeStoreFilesMetadataBatch, (o, k) -> k.writeTo(o), (o, v) -> {
+                if (v != null) {
+                    o.writeBoolean(true);
+                    v.writeTo(o);
+                } else {
+                    o.writeBoolean(false);
+                }
+            });
         }
     }
 
diff --git a/server/src/main/java/org/opensearch/ingest/CompoundProcessor.java b/server/src/main/java/org/opensearch/ingest/CompoundProcessor.java
index a5f4870029e87..64d71691bf818 100644
--- a/server/src/main/java/org/opensearch/ingest/CompoundProcessor.java
+++ b/server/src/main/java/org/opensearch/ingest/CompoundProcessor.java
@@ -39,10 +39,13 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.BiConsumer;
+import java.util.function.Consumer;
 import java.util.function.LongSupplier;
 import java.util.stream.Collectors;
 
@@ -150,6 +153,108 @@ public void execute(IngestDocument ingestDocument, BiConsumer<IngestDocument, Ex
         innerExecute(0, ingestDocument, handler);
     }
 
+    @Override
+    public void batchExecute(List<IngestDocumentWrapper> ingestDocumentWrappers, Consumer<List<IngestDocumentWrapper>> handler) {
+        innerBatchExecute(0, ingestDocumentWrappers, handler);
+    }
+
+    /**
+     * Internal logic to process documents with current processor.
+     *
+     * @param currentProcessor index of processor to process batched documents
+     * @param ingestDocumentWrappers batched documents to be processed
+     * @param handler callback function
+     */
+    void innerBatchExecute(
+        int currentProcessor,
+        List<IngestDocumentWrapper> ingestDocumentWrappers,
+        Consumer<List<IngestDocumentWrapper>> handler
+    ) {
+        if (currentProcessor == processorsWithMetrics.size()) {
+            handler.accept(ingestDocumentWrappers);
+            return;
+        }
+        Tuple<Processor, OperationMetrics> processorWithMetric = processorsWithMetrics.get(currentProcessor);
+        final Processor processor = processorWithMetric.v1();
+        final OperationMetrics metric = processorWithMetric.v2();
+        final long startTimeInNanos = relativeTimeProvider.getAsLong();
+        int size = ingestDocumentWrappers.size();
+        metric.beforeN(size);
+        // Use synchronization to ensure batches are processed by processors in sequential order
+        AtomicInteger counter = new AtomicInteger(size);
+        List<IngestDocumentWrapper> allResults = Collections.synchronizedList(new ArrayList<>());
+        Map<Integer, IngestDocumentWrapper> slotToWrapperMap = createSlotIngestDocumentWrapperMap(ingestDocumentWrappers);
+        processor.batchExecute(ingestDocumentWrappers, results -> {
+            if (results.isEmpty()) return;
+            allResults.addAll(results);
+            // counter equals to 0 means all documents are processed and called back.
+            if (counter.addAndGet(-results.size()) == 0) {
+                long ingestTimeInMillis = TimeUnit.NANOSECONDS.toMillis(relativeTimeProvider.getAsLong() - startTimeInNanos);
+                metric.afterN(allResults.size(), ingestTimeInMillis);
+
+                List<IngestDocumentWrapper> documentsDropped = new ArrayList<>();
+                List<IngestDocumentWrapper> documentsWithException = new ArrayList<>();
+                List<IngestDocumentWrapper> documentsToContinue = new ArrayList<>();
+                int totalFailed = 0;
+                // iterate all results to categorize them to: to continue, to drop, with exception
+                for (IngestDocumentWrapper resultDocumentWrapper : allResults) {
+                    IngestDocumentWrapper originalDocumentWrapper = slotToWrapperMap.get(resultDocumentWrapper.getSlot());
+                    if (resultDocumentWrapper.getException() != null) {
+                        ++totalFailed;
+                        if (ignoreFailure) {
+                            documentsToContinue.add(originalDocumentWrapper);
+                        } else {
+                            IngestProcessorException compoundProcessorException = newCompoundProcessorException(
+                                resultDocumentWrapper.getException(),
+                                processor,
+                                originalDocumentWrapper.getIngestDocument()
+                            );
+                            documentsWithException.add(
+                                new IngestDocumentWrapper(
+                                    resultDocumentWrapper.getSlot(),
+                                    originalDocumentWrapper.getIngestDocument(),
+                                    compoundProcessorException
+                                )
+                            );
+                        }
+                    } else {
+                        if (resultDocumentWrapper.getIngestDocument() == null) {
+                            documentsDropped.add(resultDocumentWrapper);
+                        } else {
+                            documentsToContinue.add(resultDocumentWrapper);
+                        }
+                    }
+                }
+                if (totalFailed > 0) {
+                    metric.failedN(totalFailed);
+                }
+                if (!documentsDropped.isEmpty()) {
+                    handler.accept(documentsDropped);
+                }
+                if (!documentsToContinue.isEmpty()) {
+                    innerBatchExecute(currentProcessor + 1, documentsToContinue, handler);
+                }
+                if (!documentsWithException.isEmpty()) {
+                    if (onFailureProcessors.isEmpty()) {
+                        handler.accept(documentsWithException);
+                    } else {
+                        documentsWithException.forEach(
+                            doc -> executeOnFailureAsync(
+                                0,
+                                doc.getIngestDocument(),
+                                (IngestProcessorException) doc.getException(),
+                                (result, ex) -> {
+                                    handler.accept(Collections.singletonList(new IngestDocumentWrapper(doc.getSlot(), result, ex)));
+                                }
+                            )
+                        );
+                    }
+                }
+            }
+            assert counter.get() >= 0;
+        });
+    }
+
     void innerExecute(int currentProcessor, IngestDocument ingestDocument, BiConsumer<IngestDocument, Exception> handler) {
         if (currentProcessor == processorsWithMetrics.size()) {
             handler.accept(ingestDocument, null);
@@ -266,4 +371,12 @@ static IngestProcessorException newCompoundProcessorException(Exception e, Proce
         return exception;
     }
 
+    private Map<Integer, IngestDocumentWrapper> createSlotIngestDocumentWrapperMap(List<IngestDocumentWrapper> ingestDocumentWrappers) {
+        Map<Integer, IngestDocumentWrapper> slotIngestDocumentWrapperMap = new HashMap<>();
+        for (IngestDocumentWrapper ingestDocumentWrapper : ingestDocumentWrappers) {
+            slotIngestDocumentWrapperMap.put(ingestDocumentWrapper.getSlot(), ingestDocumentWrapper);
+        }
+        return slotIngestDocumentWrapperMap;
+    }
+
 }
diff --git a/server/src/main/java/org/opensearch/ingest/IngestDocumentWrapper.java b/server/src/main/java/org/opensearch/ingest/IngestDocumentWrapper.java
new file mode 100644
index 0000000000000..6fb9f245f4996
--- /dev/null
+++ b/server/src/main/java/org/opensearch/ingest/IngestDocumentWrapper.java
@@ -0,0 +1,42 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.ingest;
+
+/**
+ * A IngestDocument wrapper including the slot of the IngestDocument in original IndexRequests.
+ * It also stores the exception happened during ingest process of the document.
+ */
+public final class IngestDocumentWrapper {
+    private final int slot;
+    private IngestDocument ingestDocument;
+    private Exception exception;
+
+    public IngestDocumentWrapper(int slot, IngestDocument ingestDocument, Exception ex) {
+        this.slot = slot;
+        this.ingestDocument = ingestDocument;
+        this.exception = ex;
+    }
+
+    public int getSlot() {
+        return this.slot;
+    }
+
+    public IngestDocument getIngestDocument() {
+        return this.ingestDocument;
+    }
+
+    public Exception getException() {
+        return this.exception;
+    }
+
+    public void update(IngestDocument result, Exception ex) {
+        this.ingestDocument = result;
+        this.exception = ex;
+    }
+}
diff --git a/server/src/main/java/org/opensearch/ingest/IngestService.java b/server/src/main/java/org/opensearch/ingest/IngestService.java
index 2d4439e86461b..ab8e823199447 100644
--- a/server/src/main/java/org/opensearch/ingest/IngestService.java
+++ b/server/src/main/java/org/opensearch/ingest/IngestService.java
@@ -39,6 +39,7 @@
 import org.opensearch.OpenSearchParseException;
 import org.opensearch.ResourceNotFoundException;
 import org.opensearch.action.DocWriteRequest;
+import org.opensearch.action.bulk.BulkRequest;
 import org.opensearch.action.bulk.TransportBulkAction;
 import org.opensearch.action.index.IndexRequest;
 import org.opensearch.action.ingest.DeletePipelineRequest;
@@ -93,6 +94,7 @@
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 import java.util.function.IntConsumer;
+import java.util.stream.Collectors;
 
 /**
  * Holder class for several ingest related services.
@@ -511,9 +513,9 @@ public void executeBulkRequest(
         BiConsumer<Integer, Exception> onFailure,
         BiConsumer<Thread, Exception> onCompletion,
         IntConsumer onDropped,
-        String executorName
+        String executorName,
+        BulkRequest originalBulkRequest
     ) {
-
         threadPool.executor(executorName).execute(new AbstractRunnable() {
 
             @Override
@@ -523,6 +525,12 @@ public void onFailure(Exception e) {
 
             @Override
             protected void doRun() {
+                int batchSize = originalBulkRequest.batchSize();
+                if (shouldExecuteBulkRequestInBatch(originalBulkRequest.requests().size(), batchSize)) {
+                    runBulkRequestInBatch(numberOfActionRequests, actionRequests, onFailure, onCompletion, onDropped, originalBulkRequest);
+                    return;
+                }
+
                 final Thread originalThread = Thread.currentThread();
                 final AtomicInteger counter = new AtomicInteger(numberOfActionRequests);
                 int i = 0;
@@ -536,7 +544,6 @@ protected void doRun() {
                         i++;
                         continue;
                     }
-
                     final String pipelineId = indexRequest.getPipeline();
                     indexRequest.setPipeline(NOOP_PIPELINE_NAME);
                     final String finalPipelineId = indexRequest.getFinalPipeline();
@@ -571,13 +578,286 @@ protected void doRun() {
                         onCompletion,
                         originalThread
                     );
-
                     i++;
                 }
             }
         });
     }
 
+    private void runBulkRequestInBatch(
+        int numberOfActionRequests,
+        Iterable<DocWriteRequest<?>> actionRequests,
+        BiConsumer<Integer, Exception> onFailure,
+        BiConsumer<Thread, Exception> onCompletion,
+        IntConsumer onDropped,
+        BulkRequest originalBulkRequest
+    ) {
+
+        final Thread originalThread = Thread.currentThread();
+        final AtomicInteger counter = new AtomicInteger(numberOfActionRequests);
+        int i = 0;
+        List<IndexRequestWrapper> indexRequestWrappers = new ArrayList<>();
+        for (DocWriteRequest<?> actionRequest : actionRequests) {
+            IndexRequest indexRequest = TransportBulkAction.getIndexWriteRequest(actionRequest);
+            if (indexRequest == null) {
+                if (counter.decrementAndGet() == 0) {
+                    onCompletion.accept(originalThread, null);
+                }
+                assert counter.get() >= 0;
+                i++;
+                continue;
+            }
+
+            final String pipelineId = indexRequest.getPipeline();
+            indexRequest.setPipeline(NOOP_PIPELINE_NAME);
+            final String finalPipelineId = indexRequest.getFinalPipeline();
+            indexRequest.setFinalPipeline(NOOP_PIPELINE_NAME);
+            boolean hasFinalPipeline = true;
+            final List<String> pipelines;
+            if (IngestService.NOOP_PIPELINE_NAME.equals(pipelineId) == false
+                && IngestService.NOOP_PIPELINE_NAME.equals(finalPipelineId) == false) {
+                pipelines = Arrays.asList(pipelineId, finalPipelineId);
+            } else if (IngestService.NOOP_PIPELINE_NAME.equals(pipelineId) == false) {
+                pipelines = Collections.singletonList(pipelineId);
+                hasFinalPipeline = false;
+            } else if (IngestService.NOOP_PIPELINE_NAME.equals(finalPipelineId) == false) {
+                pipelines = Collections.singletonList(finalPipelineId);
+            } else {
+                if (counter.decrementAndGet() == 0) {
+                    onCompletion.accept(originalThread, null);
+                }
+                assert counter.get() >= 0;
+                i++;
+                continue;
+            }
+
+            indexRequestWrappers.add(new IndexRequestWrapper(i, indexRequest, pipelines, hasFinalPipeline));
+            i++;
+        }
+
+        int batchSize = originalBulkRequest.batchSize();
+        List<List<IndexRequestWrapper>> batches = prepareBatches(batchSize, indexRequestWrappers);
+        logger.debug("batchSize: {}, batches: {}", batchSize, batches.size());
+
+        for (List<IndexRequestWrapper> batch : batches) {
+            executePipelinesInBatchRequests(
+                batch.stream().map(IndexRequestWrapper::getSlot).collect(Collectors.toList()),
+                batch.get(0).getPipelines().iterator(),
+                batch.get(0).isHasFinalPipeline(),
+                batch.stream().map(IndexRequestWrapper::getIndexRequest).collect(Collectors.toList()),
+                onDropped,
+                onFailure,
+                counter,
+                onCompletion,
+                originalThread
+            );
+        }
+    }
+
+    private boolean shouldExecuteBulkRequestInBatch(int documentSize, int batchSize) {
+        return documentSize > 1 && batchSize > 1;
+    }
+
+    /**
+     * IndexRequests are grouped by unique (index + pipeline_ids) before batching.
+     * Only IndexRequests in the same group could be batched. It's to ensure batched documents always
+     * flow through the same pipeline together.
+     *
+     * An IndexRequest could be preprocessed by at most two pipelines: default_pipeline and final_pipeline.
+     * A final_pipeline is configured on index level. The default_pipeline for a IndexRequest in a _bulk API
+     * could come from three places:
+     * 1. bound with index
+     * 2. a request parameter of _bulk API
+     * 3. a parameter of an IndexRequest.
+     */
+    static List<List<IndexRequestWrapper>> prepareBatches(int batchSize, List<IndexRequestWrapper> indexRequestWrappers) {
+        final Map<Integer, List<IndexRequestWrapper>> indexRequestsPerIndexAndPipelines = new HashMap<>();
+        for (IndexRequestWrapper indexRequestWrapper : indexRequestWrappers) {
+            // IndexRequests are grouped by their index + pipeline ids
+            List<String> indexAndPipelineIds = new ArrayList<>();
+            String index = indexRequestWrapper.getIndexRequest().index();
+            List<String> pipelines = indexRequestWrapper.getPipelines();
+            indexAndPipelineIds.add(index);
+            indexAndPipelineIds.addAll(pipelines);
+            int hashCode = indexAndPipelineIds.hashCode();
+            indexRequestsPerIndexAndPipelines.putIfAbsent(hashCode, new ArrayList<>());
+            indexRequestsPerIndexAndPipelines.get(hashCode).add(indexRequestWrapper);
+        }
+        List<List<IndexRequestWrapper>> batchedIndexRequests = new ArrayList<>();
+        for (Map.Entry<Integer, List<IndexRequestWrapper>> indexRequestsPerKey : indexRequestsPerIndexAndPipelines.entrySet()) {
+            for (int i = 0; i < indexRequestsPerKey.getValue().size(); i += batchSize) {
+                batchedIndexRequests.add(
+                    new ArrayList<>(
+                        indexRequestsPerKey.getValue().subList(i, i + Math.min(batchSize, indexRequestsPerKey.getValue().size() - i))
+                    )
+                );
+            }
+        }
+        return batchedIndexRequests;
+    }
+
+    /* visible for testing */
+    static final class IndexRequestWrapper {
+        private final int slot;
+        private final IndexRequest indexRequest;
+        private final List<String> pipelines;
+        private final boolean hasFinalPipeline;
+
+        IndexRequestWrapper(int slot, IndexRequest indexRequest, List<String> pipelines, boolean hasFinalPipeline) {
+            this.slot = slot;
+            this.indexRequest = indexRequest;
+            this.pipelines = pipelines;
+            this.hasFinalPipeline = hasFinalPipeline;
+        }
+
+        public int getSlot() {
+            return slot;
+        }
+
+        public IndexRequest getIndexRequest() {
+            return indexRequest;
+        }
+
+        public List<String> getPipelines() {
+            return pipelines;
+        }
+
+        public boolean isHasFinalPipeline() {
+            return hasFinalPipeline;
+        }
+    }
+
+    private void executePipelinesInBatchRequests(
+        final List<Integer> slots,
+        final Iterator<String> pipelineIterator,
+        final boolean hasFinalPipeline,
+        final List<IndexRequest> indexRequests,
+        final IntConsumer onDropped,
+        final BiConsumer<Integer, Exception> onFailure,
+        final AtomicInteger counter,
+        final BiConsumer<Thread, Exception> onCompletion,
+        final Thread originalThread
+    ) {
+        if (indexRequests.size() == 1) {
+            executePipelines(
+                slots.get(0),
+                pipelineIterator,
+                hasFinalPipeline,
+                indexRequests.get(0),
+                onDropped,
+                onFailure,
+                counter,
+                onCompletion,
+                originalThread
+            );
+            return;
+        }
+        while (pipelineIterator.hasNext()) {
+            final String pipelineId = pipelineIterator.next();
+            try {
+                PipelineHolder holder = pipelines.get(pipelineId);
+                if (holder == null) {
+                    throw new IllegalArgumentException("pipeline with id [" + pipelineId + "] does not exist");
+                }
+                Pipeline pipeline = holder.pipeline;
+                String originalIndex = indexRequests.get(0).indices()[0];
+                Map<Integer, IndexRequest> slotIndexRequestMap = createSlotIndexRequestMap(slots, indexRequests);
+                innerBatchExecute(slots, indexRequests, pipeline, onDropped, results -> {
+                    for (int i = 0; i < results.size(); ++i) {
+                        if (results.get(i).getException() != null) {
+                            IndexRequest indexRequest = slotIndexRequestMap.get(results.get(i).getSlot());
+                            logger.debug(
+                                () -> new ParameterizedMessage(
+                                    "failed to execute pipeline [{}] for document [{}/{}]",
+                                    pipelineId,
+                                    indexRequest.index(),
+                                    indexRequest.id()
+                                ),
+                                results.get(i).getException()
+                            );
+                            onFailure.accept(slots.get(i), results.get(i).getException());
+                        }
+                    }
+
+                    Iterator<String> newPipelineIterator = pipelineIterator;
+                    boolean newHasFinalPipeline = hasFinalPipeline;
+                    // indexRequests are grouped for the same index and same pipelines
+                    String newIndex = indexRequests.get(0).indices()[0];
+
+                    // handle index change case
+                    if (Objects.equals(originalIndex, newIndex) == false) {
+                        if (hasFinalPipeline && pipelineIterator.hasNext() == false) {
+                            totalMetrics.failed();
+                            for (int slot : slots) {
+                                onFailure.accept(
+                                    slot,
+                                    new IllegalStateException("final pipeline [" + pipelineId + "] can't change the target index")
+                                );
+                            }
+                        } else {
+                            // Drain old it so it's not looped over
+                            pipelineIterator.forEachRemaining($ -> {});
+                            for (IndexRequest indexRequest : indexRequests) {
+                                indexRequest.isPipelineResolved(false);
+                                resolvePipelines(null, indexRequest, state.metadata());
+                                if (IngestService.NOOP_PIPELINE_NAME.equals(indexRequest.getFinalPipeline()) == false) {
+                                    newPipelineIterator = Collections.singleton(indexRequest.getFinalPipeline()).iterator();
+                                    newHasFinalPipeline = true;
+                                } else {
+                                    newPipelineIterator = Collections.emptyIterator();
+                                }
+                            }
+                        }
+                    }
+
+                    if (newPipelineIterator.hasNext()) {
+                        executePipelinesInBatchRequests(
+                            slots,
+                            newPipelineIterator,
+                            newHasFinalPipeline,
+                            indexRequests,
+                            onDropped,
+                            onFailure,
+                            counter,
+                            onCompletion,
+                            originalThread
+                        );
+                    } else {
+                        if (counter.addAndGet(-results.size()) == 0) {
+                            onCompletion.accept(originalThread, null);
+                        }
+                        assert counter.get() >= 0;
+                    }
+                });
+            } catch (Exception e) {
+                StringBuilder documentLogBuilder = new StringBuilder();
+                for (int i = 0; i < indexRequests.size(); ++i) {
+                    IndexRequest indexRequest = indexRequests.get(i);
+                    documentLogBuilder.append(indexRequest.index());
+                    documentLogBuilder.append("/");
+                    documentLogBuilder.append(indexRequest.id());
+                    if (i < indexRequests.size() - 1) {
+                        documentLogBuilder.append(", ");
+                    }
+                    onFailure.accept(slots.get(i), e);
+                }
+                logger.debug(
+                    () -> new ParameterizedMessage(
+                        "failed to execute pipeline [{}] for documents [{}]",
+                        pipelineId,
+                        documentLogBuilder.toString()
+                    ),
+                    e
+                );
+                if (counter.addAndGet(-indexRequests.size()) == 0) {
+                    onCompletion.accept(originalThread, null);
+                }
+                assert counter.get() >= 0;
+                break;
+            }
+        }
+    }
+
     private void executePipelines(
         final int slot,
         final Iterator<String> it,
@@ -761,28 +1041,73 @@ private void innerExecute(
                 itemDroppedHandler.accept(slot);
                 handler.accept(null);
             } else {
-                Map<IngestDocument.Metadata, Object> metadataMap = ingestDocument.extractMetadata();
-                // it's fine to set all metadata fields all the time, as ingest document holds their starting values
-                // before ingestion, which might also get modified during ingestion.
-                indexRequest.index((String) metadataMap.get(IngestDocument.Metadata.INDEX));
-                indexRequest.id((String) metadataMap.get(IngestDocument.Metadata.ID));
-                indexRequest.routing((String) metadataMap.get(IngestDocument.Metadata.ROUTING));
-                indexRequest.version(((Number) metadataMap.get(IngestDocument.Metadata.VERSION)).longValue());
-                if (metadataMap.get(IngestDocument.Metadata.VERSION_TYPE) != null) {
-                    indexRequest.versionType(VersionType.fromString((String) metadataMap.get(IngestDocument.Metadata.VERSION_TYPE)));
-                }
-                if (metadataMap.get(IngestDocument.Metadata.IF_SEQ_NO) != null) {
-                    indexRequest.setIfSeqNo(((Number) metadataMap.get(IngestDocument.Metadata.IF_SEQ_NO)).longValue());
-                }
-                if (metadataMap.get(IngestDocument.Metadata.IF_PRIMARY_TERM) != null) {
-                    indexRequest.setIfPrimaryTerm(((Number) metadataMap.get(IngestDocument.Metadata.IF_PRIMARY_TERM)).longValue());
-                }
-                indexRequest.source(ingestDocument.getSourceAndMetadata(), indexRequest.getContentType());
+                updateIndexRequestWithIngestDocument(indexRequest, ingestDocument);
                 handler.accept(null);
             }
         });
     }
 
+    private void innerBatchExecute(
+        List<Integer> slots,
+        List<IndexRequest> indexRequests,
+        Pipeline pipeline,
+        IntConsumer itemDroppedHandler,
+        Consumer<List<IngestDocumentWrapper>> handler
+    ) {
+        if (pipeline.getProcessors().isEmpty()) {
+            handler.accept(null);
+            return;
+        }
+
+        int size = indexRequests.size();
+        long startTimeInNanos = System.nanoTime();
+        // the pipeline specific stat holder may not exist and that is fine:
+        // (e.g. the pipeline may have been removed while we're ingesting a document
+        totalMetrics.beforeN(size);
+        List<IngestDocumentWrapper> ingestDocumentWrappers = new ArrayList<>();
+        Map<Integer, IndexRequest> slotToindexRequestMap = new HashMap<>();
+        for (int i = 0; i < slots.size(); ++i) {
+            slotToindexRequestMap.put(slots.get(i), indexRequests.get(i));
+            ingestDocumentWrappers.add(toIngestDocumentWrapper(slots.get(i), indexRequests.get(i)));
+        }
+        AtomicInteger counter = new AtomicInteger(size);
+        List<IngestDocumentWrapper> allResults = Collections.synchronizedList(new ArrayList<>());
+        pipeline.batchExecute(ingestDocumentWrappers, results -> {
+            if (results.isEmpty()) return;
+            allResults.addAll(results);
+            if (counter.addAndGet(-results.size()) == 0) {
+                long ingestTimeInMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTimeInNanos);
+                totalMetrics.afterN(size, ingestTimeInMillis);
+                List<IngestDocumentWrapper> succeeded = new ArrayList<>();
+                List<IngestDocumentWrapper> dropped = new ArrayList<>();
+                List<IngestDocumentWrapper> exceptions = new ArrayList<>();
+                for (IngestDocumentWrapper result : allResults) {
+                    if (result.getException() != null) {
+                        exceptions.add(result);
+                    } else if (result.getIngestDocument() == null) {
+                        dropped.add(result);
+                    } else {
+                        succeeded.add(result);
+                    }
+                }
+                if (!exceptions.isEmpty()) {
+                    totalMetrics.failedN(exceptions.size());
+                } else if (!dropped.isEmpty()) {
+                    dropped.forEach(t -> itemDroppedHandler.accept(t.getSlot()));
+                } else {
+                    for (IngestDocumentWrapper ingestDocumentWrapper : succeeded) {
+                        updateIndexRequestWithIngestDocument(
+                            slotToindexRequestMap.get(ingestDocumentWrapper.getSlot()),
+                            ingestDocumentWrapper.getIngestDocument()
+                        );
+                    }
+                }
+                handler.accept(allResults);
+            }
+            assert counter.get() >= 0;
+        });
+    }
+
     @Override
     public void applyClusterState(final ClusterChangedEvent event) {
         state = event.state();
@@ -969,4 +1294,46 @@ static class PipelineHolder {
         }
     }
 
+    public static void updateIndexRequestWithIngestDocument(IndexRequest indexRequest, IngestDocument ingestDocument) {
+        Map<IngestDocument.Metadata, Object> metadataMap = ingestDocument.extractMetadata();
+        // it's fine to set all metadata fields all the time, as ingest document holds their starting values
+        // before ingestion, which might also get modified during ingestion.
+        indexRequest.index((String) metadataMap.get(IngestDocument.Metadata.INDEX));
+        indexRequest.id((String) metadataMap.get(IngestDocument.Metadata.ID));
+        indexRequest.routing((String) metadataMap.get(IngestDocument.Metadata.ROUTING));
+        indexRequest.version(((Number) metadataMap.get(IngestDocument.Metadata.VERSION)).longValue());
+        if (metadataMap.get(IngestDocument.Metadata.VERSION_TYPE) != null) {
+            indexRequest.versionType(VersionType.fromString((String) metadataMap.get(IngestDocument.Metadata.VERSION_TYPE)));
+        }
+        if (metadataMap.get(IngestDocument.Metadata.IF_SEQ_NO) != null) {
+            indexRequest.setIfSeqNo(((Number) metadataMap.get(IngestDocument.Metadata.IF_SEQ_NO)).longValue());
+        }
+        if (metadataMap.get(IngestDocument.Metadata.IF_PRIMARY_TERM) != null) {
+            indexRequest.setIfPrimaryTerm(((Number) metadataMap.get(IngestDocument.Metadata.IF_PRIMARY_TERM)).longValue());
+        }
+        indexRequest.source(ingestDocument.getSourceAndMetadata(), indexRequest.getContentType());
+    }
+
+    static IngestDocument toIngestDocument(IndexRequest indexRequest) {
+        return new IngestDocument(
+            indexRequest.index(),
+            indexRequest.id(),
+            indexRequest.routing(),
+            indexRequest.version(),
+            indexRequest.versionType(),
+            indexRequest.sourceAsMap()
+        );
+    }
+
+    private static IngestDocumentWrapper toIngestDocumentWrapper(int slot, IndexRequest indexRequest) {
+        return new IngestDocumentWrapper(slot, toIngestDocument(indexRequest), null);
+    }
+
+    private static Map<Integer, IndexRequest> createSlotIndexRequestMap(List<Integer> slots, List<IndexRequest> indexRequests) {
+        Map<Integer, IndexRequest> slotIndexRequestMap = new HashMap<>();
+        for (int i = 0; i < slots.size(); ++i) {
+            slotIndexRequestMap.put(slots.get(i), indexRequests.get(i));
+        }
+        return slotIndexRequestMap;
+    }
 }
diff --git a/server/src/main/java/org/opensearch/ingest/Pipeline.java b/server/src/main/java/org/opensearch/ingest/Pipeline.java
index 2541cfbf4af77..708416cfca3b7 100644
--- a/server/src/main/java/org/opensearch/ingest/Pipeline.java
+++ b/server/src/main/java/org/opensearch/ingest/Pipeline.java
@@ -43,6 +43,7 @@
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 import java.util.function.BiConsumer;
+import java.util.function.Consumer;
 import java.util.function.LongSupplier;
 
 /**
@@ -201,4 +202,28 @@ public List<Processor> flattenAllProcessors() {
     public OperationMetrics getMetrics() {
         return metrics;
     }
+
+    /**
+     * Modifies the data of batched multiple documents to be indexed based on the processor this pipeline holds
+     * <p>
+     * If {@code null} is returned then this document will be dropped and not indexed, otherwise
+     * this document will be kept and indexed. Document and the exception happened during processing are kept in
+     * IngestDocumentWrapper and callback to upper level.
+     *
+     * @param ingestDocumentWrappers a list of wrapped IngestDocument to ingest.
+     * @param handler callback with IngestDocument result and exception wrapped in IngestDocumentWrapper.
+     */
+    public void batchExecute(List<IngestDocumentWrapper> ingestDocumentWrappers, Consumer<List<IngestDocumentWrapper>> handler) {
+        final long startTimeInNanos = relativeTimeProvider.getAsLong();
+        int size = ingestDocumentWrappers.size();
+        metrics.beforeN(size);
+        compoundProcessor.batchExecute(ingestDocumentWrappers, results -> {
+            long ingestTimeInMillis = TimeUnit.NANOSECONDS.toMillis(relativeTimeProvider.getAsLong() - startTimeInNanos);
+            metrics.afterN(results.size(), ingestTimeInMillis);
+
+            int failedCount = (int) results.stream().filter(t -> t.getException() != null).count();
+            metrics.failedN(failedCount);
+            handler.accept(results);
+        });
+    }
 }
diff --git a/server/src/main/java/org/opensearch/ingest/Processor.java b/server/src/main/java/org/opensearch/ingest/Processor.java
index ecae1c139ea5e..9af1104502047 100644
--- a/server/src/main/java/org/opensearch/ingest/Processor.java
+++ b/server/src/main/java/org/opensearch/ingest/Processor.java
@@ -33,6 +33,7 @@
 package org.opensearch.ingest;
 
 import org.opensearch.client.Client;
+import org.opensearch.common.util.concurrent.AtomicArray;
 import org.opensearch.common.util.concurrent.ThreadContext;
 import org.opensearch.env.Environment;
 import org.opensearch.index.analysis.AnalysisRegistry;
@@ -40,7 +41,10 @@
 import org.opensearch.script.ScriptService;
 import org.opensearch.threadpool.Scheduler;
 
+import java.util.Collections;
+import java.util.List;
 import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.BiConsumer;
 import java.util.function.BiFunction;
 import java.util.function.Consumer;
@@ -81,6 +85,42 @@ default void execute(IngestDocument ingestDocument, BiConsumer<IngestDocument, E
      */
     IngestDocument execute(IngestDocument ingestDocument) throws Exception;
 
+    /**
+     * Process batched documents and they could be potentially modified by processors.
+     * Only override this method if the processor can benefit from processing documents in batches, otherwise, please
+     * use default implementation.
+     *
+     * @param ingestDocumentWrappers a list of wrapped IngestDocument
+     * @param handler callback with IngestDocument result and exception wrapped in IngestDocumentWrapper.
+     */
+    default void batchExecute(List<IngestDocumentWrapper> ingestDocumentWrappers, Consumer<List<IngestDocumentWrapper>> handler) {
+        if (ingestDocumentWrappers.isEmpty()) {
+            handler.accept(Collections.emptyList());
+            return;
+        }
+        int size = ingestDocumentWrappers.size();
+        AtomicInteger counter = new AtomicInteger(size);
+        AtomicArray<IngestDocumentWrapper> results = new AtomicArray<>(size);
+        for (int i = 0; i < size; ++i) {
+            innerExecute(i, ingestDocumentWrappers.get(i), results, counter, handler);
+        }
+    }
+
+    private void innerExecute(
+        int slot,
+        IngestDocumentWrapper ingestDocumentWrapper,
+        AtomicArray<IngestDocumentWrapper> results,
+        AtomicInteger counter,
+        Consumer<List<IngestDocumentWrapper>> handler
+    ) {
+        execute(ingestDocumentWrapper.getIngestDocument(), (doc, ex) -> {
+            results.set(slot, new IngestDocumentWrapper(ingestDocumentWrapper.getSlot(), doc, ex));
+            if (counter.decrementAndGet() == 0) {
+                handler.accept(results.asList());
+            }
+        });
+    }
+
     /**
      * Gets the type of a processor
      */
diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java
index 36c797d0dcee8..4c873f444a0c1 100644
--- a/server/src/main/java/org/opensearch/node/Node.java
+++ b/server/src/main/java/org/opensearch/node/Node.java
@@ -59,6 +59,7 @@
 import org.opensearch.client.Client;
 import org.opensearch.client.node.NodeClient;
 import org.opensearch.cluster.ClusterInfoService;
+import org.opensearch.cluster.ClusterManagerMetrics;
 import org.opensearch.cluster.ClusterModule;
 import org.opensearch.cluster.ClusterName;
 import org.opensearch.cluster.ClusterState;
@@ -136,6 +137,7 @@
 import org.opensearch.gateway.GatewayService;
 import org.opensearch.gateway.MetaStateService;
 import org.opensearch.gateway.PersistedClusterStateService;
+import org.opensearch.gateway.ShardsBatchGatewayAllocator;
 import org.opensearch.gateway.remote.RemoteClusterStateService;
 import org.opensearch.http.HttpServerTransport;
 import org.opensearch.identity.IdentityService;
@@ -146,6 +148,7 @@
 import org.opensearch.index.analysis.AnalysisRegistry;
 import org.opensearch.index.engine.EngineFactory;
 import org.opensearch.index.recovery.RemoteStoreRestoreService;
+import org.opensearch.index.remote.RemoteIndexPathUploader;
 import org.opensearch.index.remote.RemoteStoreStatsTrackerFactory;
 import org.opensearch.index.store.RemoteSegmentStoreDirectoryFactory;
 import org.opensearch.index.store.remote.filecache.FileCache;
@@ -153,6 +156,7 @@
 import org.opensearch.index.store.remote.filecache.FileCacheFactory;
 import org.opensearch.indices.IndicesModule;
 import org.opensearch.indices.IndicesService;
+import org.opensearch.indices.RemoteStoreSettings;
 import org.opensearch.indices.ShardLimitValidator;
 import org.opensearch.indices.SystemIndexDescriptor;
 import org.opensearch.indices.SystemIndices;
@@ -202,7 +206,7 @@
 import org.opensearch.plugins.ScriptPlugin;
 import org.opensearch.plugins.SearchPipelinePlugin;
 import org.opensearch.plugins.SearchPlugin;
-import org.opensearch.plugins.SecureTransportSettingsProvider;
+import org.opensearch.plugins.SecureSettingsFactory;
 import org.opensearch.plugins.SystemIndexPlugin;
 import org.opensearch.plugins.TelemetryPlugin;
 import org.opensearch.ratelimitting.admissioncontrol.AdmissionControlService;
@@ -631,12 +635,14 @@ protected Node(
             resourcesToClose.add(tracer::close);
             resourcesToClose.add(metricsRegistry::close);
 
+            final ClusterManagerMetrics clusterManagerMetrics = new ClusterManagerMetrics(metricsRegistry);
+
             List<ClusterPlugin> clusterPlugins = pluginsService.filterPlugins(ClusterPlugin.class);
             final ClusterService clusterService = new ClusterService(
                 settings,
                 settingsModule.getClusterSettings(),
                 threadPool,
-                metricsRegistry
+                clusterManagerMetrics
             );
             clusterService.addStateApplier(scriptService);
             resourcesToClose.add(clusterService);
@@ -675,7 +681,7 @@ protected Node(
                 clusterInfoService,
                 snapshotsInfoService,
                 threadPool.getThreadContext(),
-                metricsRegistry
+                clusterManagerMetrics
             );
             modules.add(clusterModule);
             IndicesModule indicesModule = new IndicesModule(pluginsService.filterPlugins(MapperPlugin.class));
@@ -731,17 +737,26 @@ protected Node(
                 threadPool::relativeTimeInMillis
             );
             final RemoteClusterStateService remoteClusterStateService;
+            final RemoteIndexPathUploader remoteIndexPathUploader;
             if (isRemoteStoreClusterStateEnabled(settings)) {
+                remoteIndexPathUploader = new RemoteIndexPathUploader(
+                    threadPool,
+                    settings,
+                    repositoriesServiceReference::get,
+                    clusterService.getClusterSettings()
+                );
                 remoteClusterStateService = new RemoteClusterStateService(
                     nodeEnvironment.nodeId(),
                     repositoriesServiceReference::get,
                     settings,
                     clusterService.getClusterSettings(),
                     threadPool::preciseRelativeTimeInNanos,
-                    threadPool
+                    threadPool,
+                    List.of(remoteIndexPathUploader)
                 );
             } else {
                 remoteClusterStateService = null;
+                remoteIndexPathUploader = null;
             }
 
             // collect engine factory providers from plugins
@@ -794,6 +809,8 @@ protected Node(
 
             final RecoverySettings recoverySettings = new RecoverySettings(settings, settingsModule.getClusterSettings());
 
+            final RemoteStoreSettings remoteStoreSettings = new RemoteStoreSettings(settings, settingsModule.getClusterSettings());
+
             final IndexStorePlugin.DirectoryFactory remoteDirectoryFactory = new RemoteSegmentStoreDirectoryFactory(
                 repositoriesServiceReference::get,
                 threadPool
@@ -831,7 +848,8 @@ protected Node(
                 searchRequestStats,
                 remoteStoreStatsTrackerFactory,
                 recoverySettings,
-                cacheService
+                cacheService,
+                remoteStoreSettings
             );
 
             final IngestService ingestService = new IngestService(
@@ -865,7 +883,8 @@ protected Node(
                 xContentRegistry,
                 systemIndices,
                 forbidPrivateIndexSettings,
-                awarenessReplicaBalance
+                awarenessReplicaBalance,
+                remoteStoreSettings
             );
             pluginsService.filterPlugins(Plugin.class)
                 .forEach(
@@ -952,9 +971,9 @@ protected Node(
                 admissionControlService
             );
 
-            final Collection<SecureTransportSettingsProvider> secureTransportSettingsProviders = pluginsService.filterPlugins(Plugin.class)
+            final Collection<SecureSettingsFactory> secureSettingsFactories = pluginsService.filterPlugins(Plugin.class)
                 .stream()
-                .map(p -> p.getSecureSettingFactory(settings).flatMap(f -> f.getSecureTransportSettingsProvider(settings)))
+                .map(p -> p.getSecureSettingFactory(settings))
                 .filter(Optional::isPresent)
                 .map(Optional::get)
                 .collect(Collectors.toList());
@@ -974,7 +993,7 @@ protected Node(
                 clusterService.getClusterSettings(),
                 tracer,
                 transportInterceptors,
-                secureTransportSettingsProviders
+                secureSettingsFactories
             );
 
             Collection<UnaryOperator<Map<String, IndexTemplateMetadata>>> indexTemplateMetadataUpgraders = pluginsService.filterPlugins(
@@ -1182,7 +1201,8 @@ protected Node(
                 resourceUsageCollectorService,
                 segmentReplicationStatsTracker,
                 repositoryService,
-                admissionControlService
+                admissionControlService,
+                cacheService
             );
 
             final SearchService searchService = newSearchService(
@@ -1314,6 +1334,7 @@ protected Node(
                 b.bind(SearchRequestSlowLog.class).toInstance(searchRequestSlowLog);
                 b.bind(MetricsRegistry.class).toInstance(metricsRegistry);
                 b.bind(RemoteClusterStateService.class).toProvider(() -> remoteClusterStateService);
+                b.bind(RemoteIndexPathUploader.class).toProvider(() -> remoteIndexPathUploader);
                 b.bind(PersistedStateRegistry.class).toInstance(persistedStateRegistry);
                 b.bind(SegmentReplicationStatsTracker.class).toInstance(segmentReplicationStatsTracker);
                 b.bind(SearchRequestOperationsCompositeListenerFactory.class).toInstance(searchRequestOperationsCompositeListenerFactory);
@@ -1323,9 +1344,12 @@ protected Node(
             // We allocate copies of existing shards by looking for a viable copy of the shard in the cluster and assigning the shard there.
             // The search for viable copies is triggered by an allocation attempt (i.e. a reroute) and is performed asynchronously. When it
             // completes we trigger another reroute to try the allocation again. This means there is a circular dependency: the allocation
-            // service needs access to the existing shards allocators (e.g. the GatewayAllocator) which need to be able to trigger a
-            // reroute, which needs to call into the allocation service. We close the loop here:
-            clusterModule.setExistingShardsAllocators(injector.getInstance(GatewayAllocator.class));
+            // service needs access to the existing shards allocators (e.g. the GatewayAllocator, ShardsBatchGatewayAllocator) which
+            // need to be able to trigger a reroute, which needs to call into the allocation service. We close the loop here:
+            clusterModule.setExistingShardsAllocators(
+                injector.getInstance(GatewayAllocator.class),
+                injector.getInstance(ShardsBatchGatewayAllocator.class)
+            );
 
             List<LifecycleComponent> pluginLifecycleComponents = pluginComponents.stream()
                 .filter(p -> p instanceof LifecycleComponent)
@@ -1463,6 +1487,10 @@ public Node start() throws NodeValidationException {
         if (remoteClusterStateService != null) {
             remoteClusterStateService.start();
         }
+        final RemoteIndexPathUploader remoteIndexPathUploader = injector.getInstance(RemoteIndexPathUploader.class);
+        if (remoteIndexPathUploader != null) {
+            remoteIndexPathUploader.start();
+        }
         // Load (and maybe upgrade) the metadata stored on disk
         final GatewayMetaState gatewayMetaState = injector.getInstance(GatewayMetaState.class);
         gatewayMetaState.start(
diff --git a/server/src/main/java/org/opensearch/node/NodeService.java b/server/src/main/java/org/opensearch/node/NodeService.java
index 15cc8f3d20bb3..1eb38ea63ad5a 100644
--- a/server/src/main/java/org/opensearch/node/NodeService.java
+++ b/server/src/main/java/org/opensearch/node/NodeService.java
@@ -41,6 +41,7 @@
 import org.opensearch.cluster.routing.WeightedRoutingStats;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.cache.service.CacheService;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.settings.SettingsFilter;
 import org.opensearch.common.util.io.IOUtils;
@@ -99,6 +100,7 @@ public class NodeService implements Closeable {
     private final RepositoriesService repositoriesService;
     private final AdmissionControlService admissionControlService;
     private final SegmentReplicationStatsTracker segmentReplicationStatsTracker;
+    private final CacheService cacheService;
 
     NodeService(
         Settings settings,
@@ -125,7 +127,8 @@ public class NodeService implements Closeable {
         ResourceUsageCollectorService resourceUsageCollectorService,
         SegmentReplicationStatsTracker segmentReplicationStatsTracker,
         RepositoriesService repositoriesService,
-        AdmissionControlService admissionControlService
+        AdmissionControlService admissionControlService,
+        CacheService cacheService
     ) {
         this.settings = settings;
         this.threadPool = threadPool;
@@ -154,6 +157,7 @@ public class NodeService implements Closeable {
         clusterService.addStateApplier(ingestService);
         clusterService.addStateApplier(searchPipelineService);
         this.segmentReplicationStatsTracker = segmentReplicationStatsTracker;
+        this.cacheService = cacheService;
     }
 
     public NodeInfo info(
@@ -236,7 +240,8 @@ public NodeStats stats(
         boolean resourceUsageStats,
         boolean segmentReplicationTrackerStats,
         boolean repositoriesStats,
-        boolean admissionControl
+        boolean admissionControl,
+        boolean cacheService
     ) {
         // for indices stats we want to include previous allocated shards stats as well (it will
         // only be applied to the sensible ones to use, like refresh/merge/flush/indexing stats)
@@ -268,7 +273,8 @@ public NodeStats stats(
             searchPipelineStats ? this.searchPipelineService.stats() : null,
             segmentReplicationTrackerStats ? this.segmentReplicationStatsTracker.getTotalRejectionStats() : null,
             repositoriesStats ? this.repositoriesService.getRepositoriesStats() : null,
-            admissionControl ? this.admissionControlService.stats() : null
+            admissionControl ? this.admissionControlService.stats() : null,
+            cacheService ? this.cacheService.stats(indices) : null
         );
     }
 
diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java
index a3bfe1195d8cc..b10ec0d99c3d5 100644
--- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java
+++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeAttribute.java
@@ -18,6 +18,7 @@
 import org.opensearch.repositories.blobstore.BlobStoreRepository;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.List;
@@ -47,6 +48,11 @@ public class RemoteStoreNodeAttribute {
     public static final String REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX = "remote_store.repository.%s.settings.";
     private final RepositoriesMetadata repositoriesMetadata;
 
+    public static List<String> SUPPORTED_DATA_REPO_NAME_ATTRIBUTES = List.of(
+        REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY,
+        REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY
+    );
+
     /**
      * Creates a new {@link RemoteStoreNodeAttribute}
      */
@@ -185,6 +191,30 @@ public RepositoriesMetadata getRepositoriesMetadata() {
         return this.repositoriesMetadata;
     }
 
+    /**
+     * Return {@link Map} of all the supported data repo names listed on {@link RemoteStoreNodeAttribute#SUPPORTED_DATA_REPO_NAME_ATTRIBUTES}
+     *
+     * @param node Node to fetch attributes from
+     * @return {@link Map} of all remote store data repo attribute keys and their values
+     */
+    public static Map<String, String> getDataRepoNames(DiscoveryNode node) {
+        assert remoteDataAttributesPresent(node.getAttributes());
+        Map<String, String> dataRepoNames = new HashMap<>();
+        for (String supportedRepoAttribute : SUPPORTED_DATA_REPO_NAME_ATTRIBUTES) {
+            dataRepoNames.put(supportedRepoAttribute, node.getAttributes().get(supportedRepoAttribute));
+        }
+        return dataRepoNames;
+    }
+
+    private static boolean remoteDataAttributesPresent(Map<String, String> nodeAttrs) {
+        for (String supportedRepoAttributes : SUPPORTED_DATA_REPO_NAME_ATTRIBUTES) {
+            if (nodeAttrs.get(supportedRepoAttributes) == null || nodeAttrs.get(supportedRepoAttributes).isEmpty()) {
+                return false;
+            }
+        }
+        return true;
+    }
+
     @Override
     public int hashCode() {
         // The hashCode is generated by computing the hash of all the repositoryMetadata present in
diff --git a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java
index 33b182dd3cc97..874c9408de6c5 100644
--- a/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java
+++ b/server/src/main/java/org/opensearch/node/remotestore/RemoteStoreNodeService.java
@@ -11,9 +11,11 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.opensearch.cluster.metadata.Metadata;
 import org.opensearch.cluster.metadata.RepositoriesMetadata;
 import org.opensearch.cluster.metadata.RepositoryMetadata;
 import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.repositories.RepositoriesService;
@@ -223,4 +225,38 @@ public RepositoriesMetadata updateRepositoriesMetadata(DiscoveryNode joiningNode
             return existingRepositories;
         }
     }
+
+    /**
+     * Returns <code>true</code> iff current cluster settings have:
+     * <br>
+     * - <code>remote_store.compatibility_mode</code> set to <code>mixed</code>
+     * <br>
+     * - <code>migration.direction</code> set to <code>remote_store</code>
+     * <br>
+     * <code>false</code> otherwise
+     *
+     * @param clusterSettings cluster level settings
+     * @return
+     * <code>true</code> For <code>REMOTE_STORE</code> migration direction and <code>MIXED</code> compatibility mode,
+     * <code>false</code> otherwise
+     */
+    public static boolean isMigratingToRemoteStore(ClusterSettings clusterSettings) {
+        boolean isMixedMode = clusterSettings.get(REMOTE_STORE_COMPATIBILITY_MODE_SETTING).equals(CompatibilityMode.MIXED);
+        boolean isRemoteStoreMigrationDirection = clusterSettings.get(MIGRATION_DIRECTION_SETTING).equals(Direction.REMOTE_STORE);
+
+        return (isMixedMode && isRemoteStoreMigrationDirection);
+    }
+
+    /**
+     * To check if the cluster is undergoing remote store migration using clusterState metadata
+     * @return
+     * <code>true</code> For <code>REMOTE_STORE</code> migration direction and <code>MIXED</code> compatibility mode,
+     * <code>false</code> otherwise
+     */
+    public static boolean isMigratingToRemoteStore(Metadata metadata) {
+        boolean isMixedMode = REMOTE_STORE_COMPATIBILITY_MODE_SETTING.get(metadata.settings()).equals(CompatibilityMode.MIXED);
+        boolean isRemoteStoreMigrationDirection = MIGRATION_DIRECTION_SETTING.get(metadata.settings()).equals(Direction.REMOTE_STORE);
+
+        return (isMixedMode && isRemoteStoreMigrationDirection);
+    }
 }
diff --git a/server/src/main/java/org/opensearch/plugins/NetworkPlugin.java b/server/src/main/java/org/opensearch/plugins/NetworkPlugin.java
index 679833c9f6e0d..138ef6f71280d 100644
--- a/server/src/main/java/org/opensearch/plugins/NetworkPlugin.java
+++ b/server/src/main/java/org/opensearch/plugins/NetworkPlugin.java
@@ -139,7 +139,7 @@ default Map<String, Supplier<HttpServerTransport>> getSecureHttpTransports(
         NetworkService networkService,
         HttpServerTransport.Dispatcher dispatcher,
         ClusterSettings clusterSettings,
-        SecureTransportSettingsProvider secureTransportSettingsProvider,
+        SecureHttpTransportSettingsProvider secureHttpTransportSettingsProvider,
         Tracer tracer
     ) {
         return Collections.emptyMap();
diff --git a/server/src/main/java/org/opensearch/plugins/SecureHttpTransportSettingsProvider.java b/server/src/main/java/org/opensearch/plugins/SecureHttpTransportSettingsProvider.java
new file mode 100644
index 0000000000000..ff86cbc04e240
--- /dev/null
+++ b/server/src/main/java/org/opensearch/plugins/SecureHttpTransportSettingsProvider.java
@@ -0,0 +1,55 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.plugins;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.http.HttpServerTransport;
+import org.opensearch.transport.TransportAdapterProvider;
+
+import javax.net.ssl.SSLEngine;
+import javax.net.ssl.SSLException;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Optional;
+
+/**
+ * A provider for security related settings for HTTP transports.
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public interface SecureHttpTransportSettingsProvider {
+    /**
+     * Collection of additional {@link TransportAdapterProvider}s that are specific to particular HTTP transport
+     * @param settings settings
+     * @return a collection of additional {@link TransportAdapterProvider}s
+     */
+    default Collection<TransportAdapterProvider<HttpServerTransport>> getHttpTransportAdapterProviders(Settings settings) {
+        return Collections.emptyList();
+    }
+
+    /**
+     * If supported, builds the {@link TransportExceptionHandler} instance for {@link HttpServerTransport} instance
+     * @param settings settings
+     * @param transport {@link HttpServerTransport} instance
+     * @return if supported, builds the {@link TransportExceptionHandler} instance
+     */
+    Optional<TransportExceptionHandler> buildHttpServerExceptionHandler(Settings settings, HttpServerTransport transport);
+
+    /**
+     * If supported, builds the {@link SSLEngine} instance for {@link HttpServerTransport} instance
+     * @param settings settings
+     * @param transport {@link HttpServerTransport} instance
+     * @return if supported, builds the {@link SSLEngine} instance
+     * @throws SSLException throws SSLException if the {@link SSLEngine} instance cannot be built
+     */
+    Optional<SSLEngine> buildSecureHttpServerEngine(Settings settings, HttpServerTransport transport) throws SSLException;
+}
diff --git a/server/src/main/java/org/opensearch/plugins/SecureSettingsFactory.java b/server/src/main/java/org/opensearch/plugins/SecureSettingsFactory.java
index b98d9cf51c129..ec2276ecc62ef 100644
--- a/server/src/main/java/org/opensearch/plugins/SecureSettingsFactory.java
+++ b/server/src/main/java/org/opensearch/plugins/SecureSettingsFactory.java
@@ -26,4 +26,11 @@ public interface SecureSettingsFactory {
      * @return optionally, the instance of the {@link SecureTransportSettingsProvider}
      */
     Optional<SecureTransportSettingsProvider> getSecureTransportSettingsProvider(Settings settings);
+
+    /**
+     * Creates (or provides pre-created) instance of the {@link SecureHttpTransportSettingsProvider}
+     * @param settings settings
+     * @return optionally, the instance of the {@link SecureHttpTransportSettingsProvider}
+     */
+    Optional<SecureHttpTransportSettingsProvider> getSecureHttpTransportSettingsProvider(Settings settings);
 }
diff --git a/server/src/main/java/org/opensearch/plugins/SecureTransportSettingsProvider.java b/server/src/main/java/org/opensearch/plugins/SecureTransportSettingsProvider.java
index 6d038ed30c8ff..5b7402a01f82d 100644
--- a/server/src/main/java/org/opensearch/plugins/SecureTransportSettingsProvider.java
+++ b/server/src/main/java/org/opensearch/plugins/SecureTransportSettingsProvider.java
@@ -10,12 +10,14 @@
 
 import org.opensearch.common.annotation.ExperimentalApi;
 import org.opensearch.common.settings.Settings;
-import org.opensearch.http.HttpServerTransport;
-import org.opensearch.transport.TcpTransport;
+import org.opensearch.transport.Transport;
+import org.opensearch.transport.TransportAdapterProvider;
 
 import javax.net.ssl.SSLEngine;
 import javax.net.ssl.SSLException;
 
+import java.util.Collection;
+import java.util.Collections;
 import java.util.Optional;
 
 /**
@@ -26,57 +28,30 @@
 @ExperimentalApi
 public interface SecureTransportSettingsProvider {
     /**
-     * An exception handler for errors that might happen while secure transport handle the requests.
-     *
-     * @see <a href="https://github.com/opensearch-project/security/blob/main/src/main/java/org/opensearch/security/ssl/SslExceptionHandler.java">SslExceptionHandler</a>
-     *
-     * @opensearch.experimental
-     */
-    @ExperimentalApi
-    @FunctionalInterface
-    interface ServerExceptionHandler {
-        static ServerExceptionHandler NOOP = t -> {};
-
-        /**
-         * Handler for errors happening during the server side processing of the requests
-         * @param t the error
-         */
-        void onError(Throwable t);
-    }
-
-    /**
-     * If supported, builds the {@link ServerExceptionHandler} instance for {@link HttpServerTransport} instance
-     * @param settings settings
-     * @param transport {@link HttpServerTransport} instance
-     * @return if supported, builds the {@link ServerExceptionHandler} instance
-     */
-    Optional<ServerExceptionHandler> buildHttpServerExceptionHandler(Settings settings, HttpServerTransport transport);
-
-    /**
-     * If supported, builds the {@link ServerExceptionHandler} instance for {@link TcpTransport} instance
+     * Collection of additional {@link TransportAdapterProvider}s that are specific to particular transport
      * @param settings settings
-     * @param transport {@link TcpTransport} instance
-     * @return if supported, builds the {@link ServerExceptionHandler} instance
+     * @return a collection of additional {@link TransportAdapterProvider}s
      */
-    Optional<ServerExceptionHandler> buildServerTransportExceptionHandler(Settings settings, TcpTransport transport);
+    default Collection<TransportAdapterProvider<Transport>> getTransportAdapterProviders(Settings settings) {
+        return Collections.emptyList();
+    }
 
     /**
-     * If supported, builds the {@link SSLEngine} instance for {@link HttpServerTransport} instance
+     * If supported, builds the {@link TransportExceptionHandler} instance for {@link Transport} instance
      * @param settings settings
-     * @param transport {@link HttpServerTransport} instance
-     * @return if supported, builds the {@link SSLEngine} instance
-     * @throws SSLException throws SSLException if the {@link SSLEngine} instance cannot be built
+     * @param transport {@link Transport} instance
+     * @return if supported, builds the {@link TransportExceptionHandler} instance
      */
-    Optional<SSLEngine> buildSecureHttpServerEngine(Settings settings, HttpServerTransport transport) throws SSLException;
+    Optional<TransportExceptionHandler> buildServerTransportExceptionHandler(Settings settings, Transport transport);
 
     /**
-     * If supported, builds the {@link SSLEngine} instance for {@link TcpTransport} instance
+     * If supported, builds the {@link SSLEngine} instance for {@link Transport} instance
      * @param settings settings
-     * @param transport {@link TcpTransport} instance
+     * @param transport {@link Transport} instance
      * @return if supported, builds the {@link SSLEngine} instance
      * @throws SSLException throws SSLException if the {@link SSLEngine} instance cannot be built
      */
-    Optional<SSLEngine> buildSecureServerTransportEngine(Settings settings, TcpTransport transport) throws SSLException;
+    Optional<SSLEngine> buildSecureServerTransportEngine(Settings settings, Transport transport) throws SSLException;
 
     /**
      * If supported, builds the {@link SSLEngine} instance for client transport instance
diff --git a/server/src/main/java/org/opensearch/plugins/TransportExceptionHandler.java b/server/src/main/java/org/opensearch/plugins/TransportExceptionHandler.java
new file mode 100644
index 0000000000000..a6b935a6b97bc
--- /dev/null
+++ b/server/src/main/java/org/opensearch/plugins/TransportExceptionHandler.java
@@ -0,0 +1,30 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.plugins;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+
+/**
+ * An exception handler for errors that might happen while secure transport handle the requests.
+ *
+ * @see <a href="https://github.com/opensearch-project/security/blob/main/src/main/java/org/opensearch/security/ssl/SslExceptionHandler.java">SslExceptionHandler</a>
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+@FunctionalInterface
+public interface TransportExceptionHandler {
+    static TransportExceptionHandler NOOP = t -> {};
+
+    /**
+     * Handler for errors happening during the server side processing of the requests
+     * @param t the error
+     */
+    void onError(Throwable t);
+}
diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BaseBlobStoreFormat.java b/server/src/main/java/org/opensearch/repositories/blobstore/BaseBlobStoreFormat.java
new file mode 100644
index 0000000000000..262a32fa1e74d
--- /dev/null
+++ b/server/src/main/java/org/opensearch/repositories/blobstore/BaseBlobStoreFormat.java
@@ -0,0 +1,130 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.repositories.blobstore;
+
+import org.apache.lucene.codecs.CodecUtil;
+import org.apache.lucene.store.OutputStreamIndexOutput;
+import org.opensearch.common.blobstore.BlobContainer;
+import org.opensearch.common.io.stream.BytesStreamOutput;
+import org.opensearch.common.lucene.store.IndexOutputOutputStream;
+import org.opensearch.common.xcontent.XContentType;
+import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.core.compress.Compressor;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
+import org.opensearch.core.xcontent.ToXContent;
+import org.opensearch.core.xcontent.XContentBuilder;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Locale;
+import java.util.Objects;
+
+/**
+ * Provides common methods, variables that can be used by the implementors.
+ *
+ * @opensearch.internal
+ */
+public class BaseBlobStoreFormat<T extends ToXContent> {
+
+    private static final int BUFFER_SIZE = 4096;
+
+    private final String blobNameFormat;
+
+    private final boolean skipHeaderFooter;
+
+    /**
+     * @param blobNameFormat format of the blobname in {@link String#format} format
+     */
+    public BaseBlobStoreFormat(String blobNameFormat, boolean skipHeaderFooter) {
+        this.blobNameFormat = blobNameFormat;
+        this.skipHeaderFooter = skipHeaderFooter;
+    }
+
+    protected String blobName(String name) {
+        return String.format(Locale.ROOT, blobNameFormat, name);
+    }
+
+    /**
+     * Writes blob with resolving the blob name using {@link #blobName} method.
+     * <p>
+     * The blob will optionally by compressed.
+     *
+     * @param obj           object to be serialized
+     * @param blobContainer blob container
+     * @param name          blob name
+     * @param compressor    whether to use compression
+     * @param params        ToXContent params
+     * @param codec         codec used
+     * @param version       version used
+     */
+    protected void write(
+        final T obj,
+        final BlobContainer blobContainer,
+        final String name,
+        final Compressor compressor,
+        final ToXContent.Params params,
+        XContentType xContentType,
+        String codec,
+        Integer version
+    ) throws IOException {
+        final String blobName = blobName(name);
+        final BytesReference bytes = serialize(obj, blobName, compressor, params, xContentType, codec, version);
+        blobContainer.writeBlob(blobName, bytes.streamInput(), bytes.length(), false);
+    }
+
+    public BytesReference serialize(
+        final T obj,
+        final String blobName,
+        final Compressor compressor,
+        final ToXContent.Params params,
+        XContentType xContentType,
+        String codec,
+        Integer version
+    ) throws IOException {
+        assert skipHeaderFooter || (Objects.nonNull(codec) && Objects.nonNull(version));
+        try (BytesStreamOutput outputStream = new BytesStreamOutput()) {
+            try (
+                OutputStreamIndexOutput indexOutput = new OutputStreamIndexOutput(
+                    "ChecksumBlobStoreFormat.writeBlob(blob=\"" + blobName + "\")",
+                    blobName,
+                    outputStream,
+                    BUFFER_SIZE
+                )
+            ) {
+                if (skipHeaderFooter == false) {
+                    CodecUtil.writeHeader(indexOutput, codec, version);
+                }
+                try (OutputStream indexOutputOutputStream = new IndexOutputOutputStream(indexOutput) {
+                    @Override
+                    public void close() {
+                        // this is important since some of the XContentBuilders write bytes on close.
+                        // in order to write the footer we need to prevent closing the actual index input.
+                    }
+                };
+                    XContentBuilder builder = MediaTypeRegistry.contentBuilder(
+                        xContentType,
+                        compressor.threadLocalOutputStream(indexOutputOutputStream)
+                    )
+                ) {
+                    builder.startObject();
+                    obj.toXContent(builder, params);
+                    builder.endObject();
+                }
+                if (skipHeaderFooter == false) {
+                    CodecUtil.writeFooter(indexOutput);
+                }
+            }
+            return outputStream.bytes();
+        }
+    }
+
+    protected String getBlobNameFormat() {
+        return blobNameFormat;
+    }
+}
diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java
index 076173177feee..1a5701d9204ef 100644
--- a/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java
+++ b/server/src/main/java/org/opensearch/repositories/blobstore/BlobStoreRepository.java
@@ -108,6 +108,7 @@
 import org.opensearch.core.xcontent.NamedXContentRegistry;
 import org.opensearch.core.xcontent.XContentParser;
 import org.opensearch.index.mapper.MapperService;
+import org.opensearch.index.remote.RemoteStorePathStrategy;
 import org.opensearch.index.snapshots.IndexShardRestoreFailedException;
 import org.opensearch.index.snapshots.IndexShardSnapshotStatus;
 import org.opensearch.index.snapshots.blobstore.BlobStoreIndexShardSnapshot;
@@ -669,7 +670,8 @@ public void cloneRemoteStoreIndexShardSnapshot(
             RemoteStoreLockManager remoteStoreMetadataLockManger = remoteStoreLockManagerFactory.newLockManager(
                 remoteStoreRepository,
                 indexUUID,
-                String.valueOf(shardId.shardId())
+                String.valueOf(shardId.shardId()),
+                remStoreBasedShardMetadata.getRemoteStorePathStrategy()
             );
             remoteStoreMetadataLockManger.cloneLock(
                 FileLockInfo.getLockInfoBuilder().withAcquirerId(source.getUUID()).build(),
@@ -1107,7 +1109,8 @@ public static void remoteDirectoryCleanupAsync(
         String remoteStoreRepoForIndex,
         String indexUUID,
         ShardId shardId,
-        String threadPoolName
+        String threadPoolName,
+        RemoteStorePathStrategy pathStrategy
     ) {
         threadpool.executor(threadPoolName)
             .execute(
@@ -1116,7 +1119,8 @@ public static void remoteDirectoryCleanupAsync(
                         remoteDirectoryFactory,
                         remoteStoreRepoForIndex,
                         indexUUID,
-                        shardId
+                        shardId,
+                        pathStrategy
                     ),
                     indexUUID,
                     shardId
@@ -1147,7 +1151,8 @@ protected void releaseRemoteStoreLockAndCleanup(
         RemoteStoreLockManager remoteStoreMetadataLockManager = remoteStoreLockManagerFactory.newLockManager(
             remoteStoreRepoForIndex,
             indexUUID,
-            shardId
+            shardId,
+            remoteStoreShardShallowCopySnapshot.getRemoteStorePathStrategy()
         );
         remoteStoreMetadataLockManager.release(FileLockInfo.getLockInfoBuilder().withAcquirerId(shallowSnapshotUUID).build());
         logger.debug("Successfully released lock for shard {} of index with uuid {}", shardId, indexUUID);
@@ -1169,7 +1174,8 @@ protected void releaseRemoteStoreLockAndCleanup(
                 remoteStoreRepoForIndex,
                 indexUUID,
                 new ShardId(Index.UNKNOWN_INDEX_NAME, indexUUID, Integer.parseInt(shardId)),
-                ThreadPool.Names.REMOTE_PURGE
+                ThreadPool.Names.REMOTE_PURGE,
+                remoteStoreShardShallowCopySnapshot.getRemoteStorePathStrategy()
             );
         }
     }
@@ -2684,6 +2690,7 @@ public void snapshotRemoteStoreIndexShard(
             // now create and write the commit point
             logger.trace("[{}] [{}] writing shard snapshot file", shardId, snapshotId);
             try {
+                RemoteStorePathStrategy pathStrategy = store.indexSettings().getRemoteStorePathStrategy();
                 REMOTE_STORE_SHARD_SHALLOW_COPY_SNAPSHOT_FORMAT.write(
                     new RemoteStoreShardShallowCopySnapshot(
                         snapshotId.getName(),
@@ -2697,7 +2704,9 @@ public void snapshotRemoteStoreIndexShard(
                         store.indexSettings().getUUID(),
                         store.indexSettings().getRemoteStoreRepository(),
                         this.basePath().toString(),
-                        fileNames
+                        fileNames,
+                        pathStrategy.getType(),
+                        pathStrategy.getHashAlgorithm()
                     ),
                     shardContainer,
                     snapshotId.getUUID(),
@@ -3153,7 +3162,7 @@ private static OffsetRangeInputStream maybeRateLimitRemoteTransfers(
     public InputStream maybeRateLimitRestores(InputStream stream) {
         return maybeRateLimit(
             maybeRateLimit(stream, () -> restoreRateLimiter, restoreRateLimitingTimeInNanos, BlobStoreTransferContext.SNAPSHOT_RESTORE),
-            recoverySettings::rateLimiter,
+            recoverySettings::recoveryRateLimiter,
             restoreRateLimitingTimeInNanos,
             BlobStoreTransferContext.SNAPSHOT_RESTORE
         );
@@ -3176,7 +3185,7 @@ public InputStream maybeRateLimitRemoteDownloadTransfers(InputStream inputStream
                 remoteDownloadRateLimitingTimeInNanos,
                 BlobStoreTransferContext.REMOTE_DOWNLOAD
             ),
-            recoverySettings::rateLimiter,
+            recoverySettings::recoveryRateLimiter,
             remoteDownloadRateLimitingTimeInNanos,
             BlobStoreTransferContext.REMOTE_DOWNLOAD
         );
diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumBlobStoreFormat.java b/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumBlobStoreFormat.java
index 3e6052a5ef820..e567e1b626c5a 100644
--- a/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumBlobStoreFormat.java
+++ b/server/src/main/java/org/opensearch/repositories/blobstore/ChecksumBlobStoreFormat.java
@@ -38,7 +38,6 @@
 import org.apache.lucene.store.ByteBuffersDataInput;
 import org.apache.lucene.store.ByteBuffersIndexInput;
 import org.apache.lucene.store.IndexInput;
-import org.apache.lucene.store.OutputStreamIndexOutput;
 import org.apache.lucene.util.BytesRef;
 import org.opensearch.cluster.metadata.Metadata;
 import org.opensearch.common.CheckedFunction;
@@ -48,26 +47,21 @@
 import org.opensearch.common.blobstore.transfer.RemoteTransferContainer;
 import org.opensearch.common.blobstore.transfer.stream.OffsetRangeIndexInputStream;
 import org.opensearch.common.io.Streams;
-import org.opensearch.common.io.stream.BytesStreamOutput;
 import org.opensearch.common.lucene.store.ByteArrayIndexInput;
-import org.opensearch.common.lucene.store.IndexOutputOutputStream;
 import org.opensearch.common.xcontent.LoggingDeprecationHandler;
 import org.opensearch.common.xcontent.XContentHelper;
 import org.opensearch.common.xcontent.XContentType;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.core.compress.Compressor;
-import org.opensearch.core.xcontent.MediaTypeRegistry;
 import org.opensearch.core.xcontent.NamedXContentRegistry;
 import org.opensearch.core.xcontent.ToXContent;
-import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.core.xcontent.XContentParser;
 import org.opensearch.gateway.CorruptStateException;
 import org.opensearch.index.store.exception.ChecksumCombinationException;
 import org.opensearch.snapshots.SnapshotInfo;
 
 import java.io.IOException;
-import java.io.OutputStream;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Locale;
@@ -80,7 +74,7 @@
  *
  * @opensearch.internal
  */
-public final class ChecksumBlobStoreFormat<T extends ToXContent> {
+public final class ChecksumBlobStoreFormat<T extends ToXContent> extends BaseBlobStoreFormat<T> {
 
     // Serialization parameters to specify correct context for metadata serialization
     public static final ToXContent.Params SNAPSHOT_ONLY_FORMAT_PARAMS;
@@ -98,12 +92,8 @@ public final class ChecksumBlobStoreFormat<T extends ToXContent> {
     // The format version
     public static final int VERSION = 1;
 
-    private static final int BUFFER_SIZE = 4096;
-
     private final String codec;
 
-    private final String blobNameFormat;
-
     private final CheckedFunction<XContentParser, T, IOException> reader;
 
     /**
@@ -112,8 +102,8 @@ public final class ChecksumBlobStoreFormat<T extends ToXContent> {
      * @param reader         prototype object that can deserialize T from XContent
      */
     public ChecksumBlobStoreFormat(String codec, String blobNameFormat, CheckedFunction<XContentParser, T, IOException> reader) {
+        super(blobNameFormat, false);
         this.reader = reader;
-        this.blobNameFormat = blobNameFormat;
         this.codec = codec;
     }
 
@@ -130,7 +120,7 @@ public T read(BlobContainer blobContainer, String name, NamedXContentRegistry na
     }
 
     public String blobName(String name) {
-        return String.format(Locale.ROOT, blobNameFormat, name);
+        return String.format(Locale.ROOT, getBlobNameFormat(), name);
     }
 
     public T deserialize(String blobName, NamedXContentRegistry namedXContentRegistry, BytesReference bytes) throws IOException {
@@ -170,30 +160,7 @@ public T deserialize(String blobName, NamedXContentRegistry namedXContentRegistr
      * @param compressor          whether to use compression
      */
     public void write(final T obj, final BlobContainer blobContainer, final String name, final Compressor compressor) throws IOException {
-        write(obj, blobContainer, name, compressor, SNAPSHOT_ONLY_FORMAT_PARAMS);
-    }
-
-    /**
-     * Writes blob with resolving the blob name using {@link #blobName} method.
-     * <p>
-     * The blob will optionally by compressed.
-     *
-     * @param obj                 object to be serialized
-     * @param blobContainer       blob container
-     * @param name                blob name
-     * @param compressor          whether to use compression
-     * @param params              ToXContent params
-     */
-    public void write(
-        final T obj,
-        final BlobContainer blobContainer,
-        final String name,
-        final Compressor compressor,
-        final ToXContent.Params params
-    ) throws IOException {
-        final String blobName = blobName(name);
-        final BytesReference bytes = serialize(obj, blobName, compressor, params);
-        blobContainer.writeBlob(blobName, bytes.streamInput(), bytes.length(), false);
+        write(obj, blobContainer, name, compressor, SNAPSHOT_ONLY_FORMAT_PARAMS, XContentType.SMILE, codec, VERSION);
     }
 
     /**
@@ -251,7 +218,7 @@ private void writeAsyncWithPriority(
         final ToXContent.Params params
     ) throws IOException {
         if (blobContainer instanceof AsyncMultiStreamBlobContainer == false) {
-            write(obj, blobContainer, name, compressor, params);
+            write(obj, blobContainer, name, compressor, params, XContentType.SMILE, codec, VERSION);
             listener.onResponse(null);
             return;
         }
@@ -290,35 +257,6 @@ private void writeAsyncWithPriority(
 
     public BytesReference serialize(final T obj, final String blobName, final Compressor compressor, final ToXContent.Params params)
         throws IOException {
-        try (BytesStreamOutput outputStream = new BytesStreamOutput()) {
-            try (
-                OutputStreamIndexOutput indexOutput = new OutputStreamIndexOutput(
-                    "ChecksumBlobStoreFormat.writeBlob(blob=\"" + blobName + "\")",
-                    blobName,
-                    outputStream,
-                    BUFFER_SIZE
-                )
-            ) {
-                CodecUtil.writeHeader(indexOutput, codec, VERSION);
-                try (OutputStream indexOutputOutputStream = new IndexOutputOutputStream(indexOutput) {
-                    @Override
-                    public void close() throws IOException {
-                        // this is important since some of the XContentBuilders write bytes on close.
-                        // in order to write the footer we need to prevent closing the actual index input.
-                    }
-                };
-                    XContentBuilder builder = MediaTypeRegistry.contentBuilder(
-                        XContentType.SMILE,
-                        compressor.threadLocalOutputStream(indexOutputOutputStream)
-                    )
-                ) {
-                    builder.startObject();
-                    obj.toXContent(builder, params);
-                    builder.endObject();
-                }
-                CodecUtil.writeFooter(indexOutput);
-            }
-            return outputStream.bytes();
-        }
+        return serialize(obj, blobName, compressor, params, XContentType.SMILE, codec, VERSION);
     }
 }
diff --git a/server/src/main/java/org/opensearch/repositories/blobstore/ConfigBlobStoreFormat.java b/server/src/main/java/org/opensearch/repositories/blobstore/ConfigBlobStoreFormat.java
new file mode 100644
index 0000000000000..18c718ca2110e
--- /dev/null
+++ b/server/src/main/java/org/opensearch/repositories/blobstore/ConfigBlobStoreFormat.java
@@ -0,0 +1,73 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.repositories.blobstore;
+
+import org.apache.lucene.store.IndexInput;
+import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer;
+import org.opensearch.common.blobstore.BlobContainer;
+import org.opensearch.common.blobstore.stream.write.WritePriority;
+import org.opensearch.common.blobstore.transfer.RemoteTransferContainer;
+import org.opensearch.common.blobstore.transfer.stream.OffsetRangeIndexInputStream;
+import org.opensearch.common.lucene.store.ByteArrayIndexInput;
+import org.opensearch.common.xcontent.XContentType;
+import org.opensearch.core.action.ActionListener;
+import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.core.compress.NoneCompressor;
+import org.opensearch.core.xcontent.ToXContent;
+
+import java.io.IOException;
+
+/**
+ * Format for writing short configurations to remote. Read interface does not exist as it not yet required. This format
+ * should be used for writing data from in-memory to remote store where there is no need for checksum and the client
+ * library for the remote store has inbuilt checksum capabilities while upload and download both. This format would
+ * serialise the data in Json format and store it on remote store as is. This does not support compression yet (this
+ * can be changed as required). In comparison to {@link ChecksumBlobStoreFormat}, this format does not add any additional
+ * metadata (like header and footer) to the content. Hence, this format does not depend on {@code CodecUtil} from
+ * Lucene library.
+ *
+ * @opensearch.internal
+ */
+public class ConfigBlobStoreFormat<T extends ToXContent> extends BaseBlobStoreFormat<T> {
+
+    /**
+     * @param blobNameFormat format of the blobname in {@link String#format} format
+     */
+    public ConfigBlobStoreFormat(String blobNameFormat) {
+        super(blobNameFormat, true);
+    }
+
+    public void writeAsyncWithUrgentPriority(T obj, BlobContainer blobContainer, String name, ActionListener<Void> listener)
+        throws IOException {
+        if (blobContainer instanceof AsyncMultiStreamBlobContainer == false) {
+            write(obj, blobContainer, name, new NoneCompressor(), ToXContent.EMPTY_PARAMS, XContentType.JSON, null, null);
+            listener.onResponse(null);
+            return;
+        }
+        String blobName = blobName(name);
+        BytesReference bytes = serialize(obj, blobName, new NoneCompressor(), ToXContent.EMPTY_PARAMS, XContentType.JSON, null, null);
+        String resourceDescription = "BlobStoreFormat.writeAsyncWithPriority(blob=\"" + blobName + "\")";
+        try (IndexInput input = new ByteArrayIndexInput(resourceDescription, BytesReference.toBytes(bytes))) {
+            try (
+                RemoteTransferContainer remoteTransferContainer = new RemoteTransferContainer(
+                    blobName,
+                    blobName,
+                    bytes.length(),
+                    true,
+                    WritePriority.URGENT,
+                    (size, position) -> new OffsetRangeIndexInputStream(input, size, position),
+                    null,
+                    false
+                )
+            ) {
+                ((AsyncMultiStreamBlobContainer) blobContainer).asyncBlobUpload(remoteTransferContainer.createWriteContext(), listener);
+            }
+        }
+    }
+}
diff --git a/server/src/main/java/org/opensearch/rest/BaseRestHandler.java b/server/src/main/java/org/opensearch/rest/BaseRestHandler.java
index e18a594236fc8..fc150405747ec 100644
--- a/server/src/main/java/org/opensearch/rest/BaseRestHandler.java
+++ b/server/src/main/java/org/opensearch/rest/BaseRestHandler.java
@@ -40,6 +40,7 @@
 import org.opensearch.action.support.clustermanager.ClusterManagerNodeRequest;
 import org.opensearch.client.node.NodeClient;
 import org.opensearch.common.CheckedConsumer;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.collect.Tuple;
 import org.opensearch.common.logging.DeprecationLogger;
 import org.opensearch.common.settings.Setting;
@@ -73,6 +74,7 @@
  *
  * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public abstract class BaseRestHandler implements RestHandler {
 
     public static final Setting<Boolean> MULTI_ALLOW_EXPLICIT_INDEX = Setting.boolSetting(
@@ -120,10 +122,6 @@ public final void handleRequest(RestRequest request, RestChannel channel, NodeCl
             throw new IllegalArgumentException(unrecognized(request, unconsumedParams, candidateParams, "parameter"));
         }
 
-        if (request.hasContent() && request.isContentConsumed() == false) {
-            throw new IllegalArgumentException("request [" + request.method() + " " + request.path() + "] does not support having a body");
-        }
-
         usageCount.increment();
         // execute the action
         action.accept(channel);
@@ -195,8 +193,11 @@ protected final String unrecognized(
     /**
      * REST requests are handled by preparing a channel consumer that represents the execution of
      * the request against a channel.
+     *
+     * @opensearch.api
      */
     @FunctionalInterface
+    @PublicApi(since = "1.0.0")
     protected interface RestChannelConsumer extends CheckedConsumer<RestChannel, Exception> {}
 
     /**
diff --git a/server/src/main/java/org/opensearch/rest/NamedRoute.java b/server/src/main/java/org/opensearch/rest/NamedRoute.java
index 109f688a4924e..c2b3ea5fdeaaf 100644
--- a/server/src/main/java/org/opensearch/rest/NamedRoute.java
+++ b/server/src/main/java/org/opensearch/rest/NamedRoute.java
@@ -9,6 +9,7 @@
 package org.opensearch.rest;
 
 import org.opensearch.OpenSearchException;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.transport.TransportService;
 
 import java.util.HashSet;
@@ -20,8 +21,9 @@
 /**
  * A named Route
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "2.7.0")
 public class NamedRoute extends RestHandler.Route {
 
     private static final String VALID_ACTION_NAME_PATTERN = "^[a-zA-Z0-9:/*_]*$";
diff --git a/server/src/main/java/org/opensearch/rest/RestChannel.java b/server/src/main/java/org/opensearch/rest/RestChannel.java
index b8ce3e92e0098..b3ded1389f754 100644
--- a/server/src/main/java/org/opensearch/rest/RestChannel.java
+++ b/server/src/main/java/org/opensearch/rest/RestChannel.java
@@ -33,6 +33,7 @@
 package org.opensearch.rest;
 
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.io.stream.BytesStreamOutput;
 import org.opensearch.core.xcontent.MediaType;
 import org.opensearch.core.xcontent.XContentBuilder;
@@ -44,6 +45,7 @@
  *
  * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface RestChannel {
 
     XContentBuilder newBuilder() throws IOException;
diff --git a/server/src/main/java/org/opensearch/rest/RestHandler.java b/server/src/main/java/org/opensearch/rest/RestHandler.java
index 294dc3ffbe329..877afdd951088 100644
--- a/server/src/main/java/org/opensearch/rest/RestHandler.java
+++ b/server/src/main/java/org/opensearch/rest/RestHandler.java
@@ -33,6 +33,7 @@
 package org.opensearch.rest;
 
 import org.opensearch.client.node.NodeClient;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.xcontent.XContent;
 import org.opensearch.rest.RestRequest.Method;
 
@@ -46,6 +47,7 @@
  *
  * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 @FunctionalInterface
 public interface RestHandler {
 
@@ -180,8 +182,9 @@ public boolean allowSystemIndexAccessByDefault() {
     /**
      * Route for the request.
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     class Route {
 
         protected final String path;
@@ -231,7 +234,10 @@ public boolean equals(Object o) {
 
     /**
      * Represents an API that has been deprecated and is slated for removal.
+     *
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     class DeprecatedRoute extends Route {
 
         private final String deprecationMessage;
@@ -249,7 +255,10 @@ public String getDeprecationMessage() {
     /**
      * Represents an API that has had its {@code path} or {@code method} changed. Holds both the
      * new and previous {@code path} and {@code method} combination.
+     *
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     class ReplacedRoute extends Route {
 
         private final String deprecatedPath;
diff --git a/server/src/main/java/org/opensearch/rest/RestResponse.java b/server/src/main/java/org/opensearch/rest/RestResponse.java
index 2eff746e8508c..482eb6b052e9b 100644
--- a/server/src/main/java/org/opensearch/rest/RestResponse.java
+++ b/server/src/main/java/org/opensearch/rest/RestResponse.java
@@ -33,6 +33,7 @@
 package org.opensearch.rest;
 
 import org.opensearch.OpenSearchException;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.lease.Releasable;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.core.rest.RestStatus;
@@ -49,6 +50,7 @@
  *
  * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public abstract class RestResponse {
 
     private Map<String, List<String>> customHeaders;
diff --git a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestNodesStatsAction.java b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestNodesStatsAction.java
index 66b9afda06eb6..267bfde576dec 100644
--- a/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestNodesStatsAction.java
+++ b/server/src/main/java/org/opensearch/rest/action/admin/cluster/RestNodesStatsAction.java
@@ -36,6 +36,7 @@
 import org.opensearch.action.admin.indices.stats.CommonStatsFlags;
 import org.opensearch.action.admin.indices.stats.CommonStatsFlags.Flag;
 import org.opensearch.client.node.NodeClient;
+import org.opensearch.common.cache.CacheType;
 import org.opensearch.core.common.Strings;
 import org.opensearch.rest.BaseRestHandler;
 import org.opensearch.rest.RestRequest;
@@ -175,6 +176,25 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
 
                     nodesStatsRequest.indices(flags);
                 }
+            } else if (metrics.contains("caches")) {
+                // Extract the list of caches we want to get stats for from the submetrics (which we get from index_metric)
+                Set<String> cacheMetrics = Strings.tokenizeByCommaToSet(request.param("index_metric", "_all"));
+                CommonStatsFlags cacheFlags = new CommonStatsFlags();
+                cacheFlags.clear();
+                if (cacheMetrics.contains("_all")) {
+                    cacheFlags.includeAllCacheTypes();
+                } else {
+                    for (String cacheName : cacheMetrics) {
+                        try {
+                            cacheFlags.includeCacheType(CacheType.getByValue(cacheName));
+                        } catch (IllegalArgumentException e) {
+                            throw new IllegalArgumentException(
+                                unrecognized(request, Set.of(cacheName), CacheType.allValues(), "cache type")
+                            );
+                        }
+                    }
+                }
+                nodesStatsRequest.indices(cacheFlags);
             } else if (request.hasParam("index_metric")) {
                 throw new IllegalArgumentException(
                     String.format(
@@ -209,6 +229,10 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
             nodesStatsRequest.indices().includeOnlyTopIndexingPressureMetrics(request.paramAsBoolean("top", false));
         }
 
+        // If no levels are passed in this results in an empty array.
+        String[] levels = Strings.splitStringByCommaToArray(request.param("level"));
+        nodesStatsRequest.indices().setLevels(levels);
+
         return channel -> client.admin().cluster().nodesStats(nodesStatsRequest, new NodesResponseRestListener<>(channel));
     }
 
diff --git a/server/src/main/java/org/opensearch/rest/action/cat/RestShardsAction.java b/server/src/main/java/org/opensearch/rest/action/cat/RestShardsAction.java
index 4cd10c6874e0a..4413c8eb370be 100644
--- a/server/src/main/java/org/opensearch/rest/action/cat/RestShardsAction.java
+++ b/server/src/main/java/org/opensearch/rest/action/cat/RestShardsAction.java
@@ -253,6 +253,10 @@ protected Table getTableWithHeader(final RestRequest request) {
             "search.point_in_time_total",
             "alias:spto,searchPointInTimeTotal;default:false;text-align:right;desc:completed point in time contexts"
         );
+        table.addCell(
+            "search.search_idle_reactivate_count_total",
+            "alias:ssirct,searchSearchIdleReactivateCountTotal;default:false;text-align:right;desc:number of times a shard reactivated"
+        );
 
         table.addCell("segments.count", "alias:sc,segmentsCount;default:false;text-align:right;desc:number of segments");
         table.addCell("segments.memory", "alias:sm,segmentsMemory;default:false;text-align:right;desc:memory used by segments");
@@ -427,6 +431,7 @@ Table buildTable(RestRequest request, ClusterStateResponse state, IndicesStatsRe
             table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getPitCurrent()));
             table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getPitTime()));
             table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getPitCount()));
+            table.addCell(getOrNull(commonStats, CommonStats::getSearch, i -> i.getTotal().getSearchIdleReactivateCount()));
 
             table.addCell(getOrNull(commonStats, CommonStats::getSegments, SegmentsStats::getCount));
             table.addCell(getOrNull(commonStats, CommonStats::getSegments, SegmentsStats::getZeroMemory));
diff --git a/server/src/main/java/org/opensearch/rest/action/document/RestBulkAction.java b/server/src/main/java/org/opensearch/rest/action/document/RestBulkAction.java
index b046146707885..0bc4234c9b8b8 100644
--- a/server/src/main/java/org/opensearch/rest/action/document/RestBulkAction.java
+++ b/server/src/main/java/org/opensearch/rest/action/document/RestBulkAction.java
@@ -97,6 +97,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
         Boolean defaultRequireAlias = request.paramAsBoolean(DocWriteRequest.REQUIRE_ALIAS, null);
         bulkRequest.timeout(request.paramAsTime("timeout", BulkShardRequest.DEFAULT_TIMEOUT));
         bulkRequest.setRefreshPolicy(request.param("refresh"));
+        bulkRequest.batchSize(request.paramAsInt("batch_size", 1));
         bulkRequest.add(
             request.requiredContent(),
             defaultIndex,
diff --git a/server/src/main/java/org/opensearch/rest/action/search/RestSearchAction.java b/server/src/main/java/org/opensearch/rest/action/search/RestSearchAction.java
index 80dc34c4d5d68..3a6b45013e892 100644
--- a/server/src/main/java/org/opensearch/rest/action/search/RestSearchAction.java
+++ b/server/src/main/java/org/opensearch/rest/action/search/RestSearchAction.java
@@ -50,6 +50,7 @@
 import org.opensearch.rest.action.RestCancellableNodeClient;
 import org.opensearch.rest.action.RestStatusToXContentListener;
 import org.opensearch.search.Scroll;
+import org.opensearch.search.SearchService;
 import org.opensearch.search.builder.SearchSourceBuilder;
 import org.opensearch.search.fetch.StoredFieldsContext;
 import org.opensearch.search.fetch.subphase.FetchSourceContext;
@@ -235,13 +236,12 @@ private static void parseSearchSource(final SearchSourceBuilder searchSourceBuil
             searchSourceBuilder.query(queryBuilder);
         }
 
-        int from = request.paramAsInt("from", -1);
-        if (from != -1) {
-            searchSourceBuilder.from(from);
+        if (request.hasParam("from")) {
+            searchSourceBuilder.from(request.paramAsInt("from", SearchService.DEFAULT_FROM));
         }
-        int size = request.paramAsInt("size", -1);
-        if (size != -1) {
-            setSize.accept(size);
+
+        if (request.hasParam("size")) {
+            setSize.accept(request.paramAsInt("size", SearchService.DEFAULT_SIZE));
         }
 
         if (request.hasParam("explain")) {
diff --git a/server/src/main/java/org/opensearch/script/DerivedFieldScript.java b/server/src/main/java/org/opensearch/script/DerivedFieldScript.java
new file mode 100644
index 0000000000000..0a2b7cf691283
--- /dev/null
+++ b/server/src/main/java/org/opensearch/script/DerivedFieldScript.java
@@ -0,0 +1,150 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.script;
+
+import org.apache.lucene.index.LeafReaderContext;
+import org.opensearch.common.collect.Tuple;
+import org.opensearch.index.fielddata.ScriptDocValues;
+import org.opensearch.search.lookup.LeafSearchLookup;
+import org.opensearch.search.lookup.SearchLookup;
+import org.opensearch.search.lookup.SourceLookup;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.Function;
+
+/**
+ * Definition of Script for DerivedField.
+ * It will be used to execute scripts defined against derived fields of any type
+ *
+ * @opensearch.internal
+ */
+public abstract class DerivedFieldScript {
+
+    public static final String[] PARAMETERS = {};
+    public static final ScriptContext<Factory> CONTEXT = new ScriptContext<>("derived_field", Factory.class);
+    private static final int MAX_BYTE_SIZE = 1024 * 1024; // Maximum allowed byte size (1 MB)
+
+    private static final Map<String, Function<Object, Object>> PARAMS_FUNCTIONS = Map.of(
+        "doc",
+        value -> value,
+        "_source",
+        value -> ((SourceLookup) value).loadSourceIfNeeded()
+    );
+
+    /**
+     * The generic runtime parameters for the script.
+     */
+    private final Map<String, Object> params;
+
+    /**
+     * A leaf lookup for the bound segment this script will operate on.
+     */
+    private final LeafSearchLookup leafLookup;
+
+    /**
+     * The field values emitted from the script.
+     */
+    private List<Object> emittedValues;
+
+    private int totalByteSize;
+
+    public DerivedFieldScript(Map<String, Object> params, SearchLookup lookup, LeafReaderContext leafContext) {
+        Map<String, Object> parameters = new HashMap<>(params);
+        this.leafLookup = lookup.getLeafSearchLookup(leafContext);
+        parameters.putAll(leafLookup.asMap());
+        this.params = new DynamicMap(parameters, PARAMS_FUNCTIONS);
+        this.emittedValues = new ArrayList<>();
+        this.totalByteSize = 0;
+    }
+
+    /**
+     * Return the parameters for this script.
+     */
+    public Map<String, Object> getParams() {
+        return params;
+    }
+
+    /**
+     * The doc lookup for the Lucene segment this script was created for.
+     */
+    public Map<String, ScriptDocValues<?>> getDoc() {
+        return leafLookup.doc();
+    }
+
+    /**
+     * Return the emitted values from the script execution.
+     */
+    public List<Object> getEmittedValues() {
+        return emittedValues;
+    }
+
+    /**
+     * Set the current document to run the script on next.
+     * Clears the emittedValues as well since they should be scoped per document.
+     */
+    public void setDocument(int docid) {
+        this.emittedValues = new ArrayList<>();
+        this.totalByteSize = 0;
+        leafLookup.setDocument(docid);
+    }
+
+    public void addEmittedValue(Object o) {
+        int byteSize = getObjectByteSize(o);
+        int newTotalByteSize = totalByteSize + byteSize;
+        if (newTotalByteSize <= MAX_BYTE_SIZE) {
+            emittedValues.add(o);
+            totalByteSize = newTotalByteSize;
+        } else {
+            throw new IllegalStateException("Exceeded maximum allowed byte size for emitted values");
+        }
+    }
+
+    private int getObjectByteSize(Object obj) {
+        if (obj instanceof String) {
+            return ((String) obj).getBytes(StandardCharsets.UTF_8).length;
+        } else if (obj instanceof Integer) {
+            return Integer.BYTES;
+        } else if (obj instanceof Long) {
+            return Long.BYTES;
+        } else if (obj instanceof Double) {
+            return Double.BYTES;
+        } else if (obj instanceof Boolean) {
+            return Byte.BYTES; // Assuming 1 byte for boolean
+        } else if (obj instanceof Tuple) {
+            // Assuming each element in the tuple is a double for GeoPoint case
+            return Double.BYTES * 2;
+        } else {
+            throw new IllegalArgumentException("Unsupported object type passed in emit()");
+        }
+    }
+
+    public void execute() {}
+
+    /**
+     * A factory to construct {@link DerivedFieldScript} instances.
+     *
+     * @opensearch.internal
+     */
+    public interface LeafFactory {
+        DerivedFieldScript newInstance(LeafReaderContext ctx) throws IOException;
+    }
+
+    /**
+     * A factory to construct stateful {@link DerivedFieldScript} factories for a specific index.
+     * @opensearch.internal
+     */
+    public interface Factory extends ScriptFactory {
+        LeafFactory newFactory(Map<String, Object> params, SearchLookup lookup);
+    }
+}
diff --git a/server/src/main/java/org/opensearch/script/ScriptEmitValues.java b/server/src/main/java/org/opensearch/script/ScriptEmitValues.java
new file mode 100644
index 0000000000000..5d12f36442179
--- /dev/null
+++ b/server/src/main/java/org/opensearch/script/ScriptEmitValues.java
@@ -0,0 +1,63 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.script;
+
+import org.opensearch.common.collect.Tuple;
+
+/**
+ * Values that can be emitted in a derived field script context.
+ * <p>
+ * The emit function can be called multiple times within a script definition
+ * so the function will handle collecting the values over the script execution.
+ */
+public final class ScriptEmitValues {
+
+    /**
+     * Takes in a single value and emits it
+     * Could be a long, double, String, etc.
+     */
+    public static final class EmitSingle {
+
+        private final DerivedFieldScript derivedFieldScript;
+
+        public EmitSingle(DerivedFieldScript derivedFieldScript) {
+            this.derivedFieldScript = derivedFieldScript;
+        }
+
+        // TODO: Keeping this generic for the time being due to limitations with
+        // binding methods with the same name and arity.
+        // Ideally, we should have an emit signature per derived field type and try to scope
+        // that to the respective script execution so the other emits aren't allowed.
+        // One way to do this could be to create implementations of the DerivedFieldScript.LeafFactory
+        // per field type where they each define their own emit() method and then the engine that executes
+        // it can have custom compilation logic to perform class bindings on that emit implementation.
+        public void emit(Object val) {
+            derivedFieldScript.addEmittedValue(val);
+        }
+
+    }
+
+    /**
+     * Emits a GeoPoint value
+     */
+    public static final class GeoPoint {
+
+        private final DerivedFieldScript derivedFieldScript;
+
+        public GeoPoint(DerivedFieldScript derivedFieldScript) {
+            this.derivedFieldScript = derivedFieldScript;
+        }
+
+        public void emit(double lat, double lon) {
+            derivedFieldScript.addEmittedValue(new Tuple<>(lat, lon));
+        }
+
+    }
+
+}
diff --git a/server/src/main/java/org/opensearch/script/ScriptModule.java b/server/src/main/java/org/opensearch/script/ScriptModule.java
index a192e9553016b..c83a6e64d53eb 100644
--- a/server/src/main/java/org/opensearch/script/ScriptModule.java
+++ b/server/src/main/java/org/opensearch/script/ScriptModule.java
@@ -78,7 +78,8 @@ public class ScriptModule {
             ScriptedMetricAggContexts.MapScript.CONTEXT,
             ScriptedMetricAggContexts.CombineScript.CONTEXT,
             ScriptedMetricAggContexts.ReduceScript.CONTEXT,
-            IntervalFilterScript.CONTEXT
+            IntervalFilterScript.CONTEXT,
+            DerivedFieldScript.CONTEXT
         ).collect(Collectors.toMap(c -> c.name, Function.identity()));
     }
 
diff --git a/server/src/main/java/org/opensearch/search/DefaultSearchContext.java b/server/src/main/java/org/opensearch/search/DefaultSearchContext.java
index 061aa2f6e5896..cd8714f6b556a 100644
--- a/server/src/main/java/org/opensearch/search/DefaultSearchContext.java
+++ b/server/src/main/java/org/opensearch/search/DefaultSearchContext.java
@@ -107,6 +107,7 @@
 import java.util.function.LongSupplier;
 
 import static org.opensearch.search.SearchService.CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING;
+import static org.opensearch.search.SearchService.MAX_AGGREGATION_REWRITE_FILTERS;
 
 /**
  * The main search context used during search phase
@@ -187,6 +188,7 @@ final class DefaultSearchContext extends SearchContext {
     private final Function<SearchSourceBuilder, InternalAggregation.ReduceContextBuilder> requestToAggReduceContextBuilder;
     private final boolean concurrentSearchSettingsEnabled;
     private final SetOnce<Boolean> requestShouldUseConcurrentSearch = new SetOnce<>();
+    private final int maxAggRewriteFilters;
 
     DefaultSearchContext(
         ReaderContext readerContext,
@@ -240,6 +242,8 @@ final class DefaultSearchContext extends SearchContext {
         queryBoost = request.indexBoost();
         this.lowLevelCancellation = lowLevelCancellation;
         this.requestToAggReduceContextBuilder = requestToAggReduceContextBuilder;
+
+        this.maxAggRewriteFilters = evaluateFilterRewriteSetting();
     }
 
     @Override
@@ -962,6 +966,12 @@ public BucketCollectorProcessor bucketCollectorProcessor() {
      *         false: otherwise
      */
     private boolean evaluateConcurrentSegmentSearchSettings(Executor concurrentSearchExecutor) {
+        // Do not use concurrent segment search for system indices or throttled requests. See:
+        // https://github.com/opensearch-project/OpenSearch/issues/12951
+        if (indexShard.isSystem() || indexShard.indexSettings().isSearchThrottled()) {
+            return false;
+        }
+
         if ((clusterService != null) && (concurrentSearchExecutor != null)) {
             return indexService.getIndexSettings()
                 .getSettings()
@@ -969,9 +979,8 @@ private boolean evaluateConcurrentSegmentSearchSettings(Executor concurrentSearc
                     IndexSettings.INDEX_CONCURRENT_SEGMENT_SEARCH_SETTING.getKey(),
                     clusterService.getClusterSettings().get(CLUSTER_CONCURRENT_SEGMENT_SEARCH_SETTING)
                 );
-        } else {
-            return false;
         }
+        return false;
     }
 
     @Override
@@ -989,4 +998,16 @@ public boolean shouldUseTimeSeriesDescSortOptimization() {
             && sort.isSortOnTimeSeriesField()
             && sort.sort.getSort()[0].getReverse() == false;
     }
+
+    @Override
+    public int maxAggRewriteFilters() {
+        return maxAggRewriteFilters;
+    }
+
+    private int evaluateFilterRewriteSetting() {
+        if (clusterService != null) {
+            return clusterService.getClusterSettings().get(MAX_AGGREGATION_REWRITE_FILTERS);
+        }
+        return 0;
+    }
 }
diff --git a/server/src/main/java/org/opensearch/search/SearchService.java b/server/src/main/java/org/opensearch/search/SearchService.java
index 62eb597e387e6..744d3a19f1593 100644
--- a/server/src/main/java/org/opensearch/search/SearchService.java
+++ b/server/src/main/java/org/opensearch/search/SearchService.java
@@ -77,6 +77,9 @@
 import org.opensearch.index.IndexService;
 import org.opensearch.index.IndexSettings;
 import org.opensearch.index.engine.Engine;
+import org.opensearch.index.mapper.DerivedField;
+import org.opensearch.index.mapper.DerivedFieldMapper;
+import org.opensearch.index.mapper.MappedFieldType;
 import org.opensearch.index.query.InnerHitContextBuilder;
 import org.opensearch.index.query.MatchAllQueryBuilder;
 import org.opensearch.index.query.MatchNoneQueryBuilder;
@@ -269,6 +272,15 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
         Property.NodeScope
     );
 
+    // value 0 means rewrite filters optimization in aggregations will be disabled
+    public static final Setting<Integer> MAX_AGGREGATION_REWRITE_FILTERS = Setting.intSetting(
+        "search.max_aggregation_rewrite_filters",
+        3000,
+        0,
+        Property.Dynamic,
+        Property.NodeScope
+    );
+
     public static final int DEFAULT_SIZE = 10;
     public static final int DEFAULT_FROM = 0;
 
@@ -1066,6 +1078,28 @@ private DefaultSearchContext createSearchContext(ReaderContext reader, ShardSear
             // might end up with incorrect state since we are using now() or script services
             // during rewrite and normalized / evaluate templates etc.
             QueryShardContext context = new QueryShardContext(searchContext.getQueryShardContext());
+            if (request.source() != null
+                && request.source().size() != 0
+                && (request.source().getDerivedFieldsObject() != null || request.source().getDerivedFields() != null)) {
+                Map<String, MappedFieldType> derivedFieldTypeMap = new HashMap<>();
+                if (request.source().getDerivedFieldsObject() != null) {
+                    Map<String, Object> derivedFieldObject = new HashMap<>();
+                    derivedFieldObject.put(DerivedFieldMapper.CONTENT_TYPE, request.source().getDerivedFieldsObject());
+                    derivedFieldTypeMap.putAll(
+                        DerivedFieldMapper.getAllDerivedFieldTypeFromObject(derivedFieldObject, searchContext.mapperService())
+                    );
+                }
+                if (request.source().getDerivedFields() != null) {
+                    for (DerivedField derivedField : request.source().getDerivedFields()) {
+                        derivedFieldTypeMap.put(
+                            derivedField.getName(),
+                            DerivedFieldMapper.getDerivedFieldType(derivedField, searchContext.mapperService())
+                        );
+                    }
+                }
+                context.setDerivedFieldTypes(derivedFieldTypeMap);
+                searchContext.getQueryShardContext().setDerivedFieldTypes(derivedFieldTypeMap);
+            }
             Rewriteable.rewrite(request.getRewriteable(), context, true);
             assert searchContext.getQueryShardContext().isCacheable();
             success = true;
diff --git a/server/src/main/java/org/opensearch/search/aggregations/AggregationCollectorManager.java b/server/src/main/java/org/opensearch/search/aggregations/AggregationCollectorManager.java
index 0bb2d1d7ca933..94e9ce5063277 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/AggregationCollectorManager.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/AggregationCollectorManager.java
@@ -15,9 +15,9 @@
 import org.opensearch.search.query.ReduceableSearchResult;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
+import java.util.Objects;
 
 /**
  * Common {@link CollectorManager} used by both concurrent and non-concurrent aggregation path and also for global and non-global
@@ -56,17 +56,9 @@ public String getCollectorReason() {
 
     @Override
     public ReduceableSearchResult reduce(Collection<Collector> collectors) throws IOException {
-        final List<Aggregator> aggregators = context.bucketCollectorProcessor().toAggregators(collectors);
-        final List<InternalAggregation> internals = new ArrayList<>(aggregators.size());
+        final List<InternalAggregation> internals = context.bucketCollectorProcessor().toInternalAggregations(collectors);
+        assert internals.stream().noneMatch(Objects::isNull);
         context.aggregations().resetBucketMultiConsumer();
-        for (Aggregator aggregator : aggregators) {
-            try {
-                // post collection is called in ContextIndexSearcher after search on leaves are completed
-                internals.add(aggregator.buildTopLevel());
-            } catch (IOException e) {
-                throw new AggregationExecutionException("Failed to build aggregation [" + aggregator.name() + "]", e);
-            }
-        }
 
         final InternalAggregations internalAggregations = InternalAggregations.from(internals);
         return buildAggregationResult(internalAggregations);
diff --git a/server/src/main/java/org/opensearch/search/aggregations/Aggregator.java b/server/src/main/java/org/opensearch/search/aggregations/Aggregator.java
index 8744d1f6a07d3..f4db8f61bf537 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/Aggregator.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/Aggregator.java
@@ -33,6 +33,7 @@
 package org.opensearch.search.aggregations;
 
 import org.opensearch.OpenSearchParseException;
+import org.opensearch.common.SetOnce;
 import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.lease.Releasable;
 import org.opensearch.core.ParseField;
@@ -61,6 +62,8 @@
 @PublicApi(since = "1.0.0")
 public abstract class Aggregator extends BucketCollector implements Releasable {
 
+    private final SetOnce<InternalAggregation> internalAggregation = new SetOnce<>();
+
     /**
      * Parses the aggregation request and creates the appropriate aggregator factory for it.
      *
@@ -83,6 +86,13 @@ public interface Parser {
         AggregationBuilder parse(String aggregationName, XContentParser parser) throws IOException;
     }
 
+    /**
+     * Returns the InternalAggregation stored during post collection
+     */
+    public InternalAggregation getPostCollectionAggregation() {
+        return internalAggregation.get();
+    }
+
     /**
      * Return the name of this aggregator.
      */
@@ -185,13 +195,15 @@ public interface BucketComparator {
 
     /**
      * Build the result of this aggregation if it is at the "top level"
-     * of the aggregation tree. If, instead, it is a sub-aggregation of
-     * another aggregation then the aggregation that contains it will call
-     * {@link #buildAggregations(long[])}.
+     * of the aggregation tree and save it. This should get called
+     * during post collection. If, instead, it is a sub-aggregation
+     * of another aggregation then the aggregation that contains
+     * it will call {@link #buildAggregations(long[])}.
      */
     public final InternalAggregation buildTopLevel() throws IOException {
         assert parent() == null;
-        return buildAggregations(new long[] { 0 })[0];
+        this.internalAggregation.set(buildAggregations(new long[] { 0 })[0]);
+        return internalAggregation.get();
     }
 
     /**
diff --git a/server/src/main/java/org/opensearch/search/aggregations/BucketCollectorProcessor.java b/server/src/main/java/org/opensearch/search/aggregations/BucketCollectorProcessor.java
index 135fda71a757a..df05ce3f5c049 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/BucketCollectorProcessor.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/BucketCollectorProcessor.java
@@ -72,6 +72,15 @@ public void processPostCollection(Collector collectorTree) throws IOException {
                 }
             } else if (currentCollector instanceof BucketCollector) {
                 ((BucketCollector) currentCollector).postCollection();
+
+                // Perform build aggregation during post collection
+                if (currentCollector instanceof Aggregator) {
+                    ((Aggregator) currentCollector).buildTopLevel();
+                } else if (currentCollector instanceof MultiBucketCollector) {
+                    for (Collector innerCollector : ((MultiBucketCollector) currentCollector).getCollectors()) {
+                        collectors.offer(innerCollector);
+                    }
+                }
             }
         }
     }
@@ -106,4 +115,31 @@ public List<Aggregator> toAggregators(Collection<Collector> collectors) {
         }
         return aggregators;
     }
+
+    /**
+     * Unwraps the input collection of {@link Collector} to get the list of the {@link InternalAggregation}. The
+     * input is expected to contain the collectors related to Aggregations only as that is passed to {@link AggregationCollectorManager}
+     * during the reduce phase. This list of {@link InternalAggregation} is used to optionally perform reduce at shard level before
+     * returning response to coordinator
+     * @param collectors collection of aggregation collectors to reduce
+     * @return list of unwrapped {@link InternalAggregation}
+     */
+    public List<InternalAggregation> toInternalAggregations(Collection<Collector> collectors) throws IOException {
+        List<InternalAggregation> internalAggregations = new ArrayList<>();
+
+        final Deque<Collector> allCollectors = new LinkedList<>(collectors);
+        while (!allCollectors.isEmpty()) {
+            Collector currentCollector = allCollectors.pop();
+            if (currentCollector instanceof InternalProfileCollector) {
+                currentCollector = ((InternalProfileCollector) currentCollector).getCollector();
+            }
+
+            if (currentCollector instanceof Aggregator) {
+                internalAggregations.add(((Aggregator) currentCollector).getPostCollectionAggregation());
+            } else if (currentCollector instanceof MultiBucketCollector) {
+                allCollectors.addAll(Arrays.asList(((MultiBucketCollector) currentCollector).getCollectors()));
+            }
+        }
+        return internalAggregations;
+    }
 }
diff --git a/server/src/main/java/org/opensearch/search/aggregations/GlobalAggCollectorManager.java b/server/src/main/java/org/opensearch/search/aggregations/GlobalAggCollectorManager.java
index 8814cc3c435e1..db2a11b47fbc3 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/GlobalAggCollectorManager.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/GlobalAggCollectorManager.java
@@ -12,8 +12,10 @@
 import org.apache.lucene.search.CollectorManager;
 import org.opensearch.search.internal.SearchContext;
 import org.opensearch.search.profile.query.CollectorResult;
+import org.opensearch.search.query.ReduceableSearchResult;
 
 import java.io.IOException;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Objects;
 
@@ -42,6 +44,19 @@ public Collector newCollector() throws IOException {
         }
     }
 
+    @Override
+    public ReduceableSearchResult reduce(Collection<Collector> collectors) throws IOException {
+        // If there are no leaves then in concurrent search case postCollection, and subsequently buildAggregation, will not be called in
+        // search path. Since we build the InternalAggregation in postCollection that will not get created in such cases either. Therefore
+        // we need to manually processPostCollection here to build empty InternalAggregation objects for this collector tree.
+        if (context.searcher().getLeafContexts().isEmpty()) {
+            for (Collector c : collectors) {
+                context.bucketCollectorProcessor().processPostCollection(c);
+            }
+        }
+        return super.reduce(collectors);
+    }
+
     @Override
     protected AggregationReduceableSearchResult buildAggregationResult(InternalAggregations internalAggregations) {
         // Reduce the aggregations across slices before sending to the coordinator. We will perform shard level reduce as long as any slices
diff --git a/server/src/main/java/org/opensearch/search/aggregations/MultiBucketConsumerService.java b/server/src/main/java/org/opensearch/search/aggregations/MultiBucketConsumerService.java
index 1461dd3009b44..35186422fceaa 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/MultiBucketConsumerService.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/MultiBucketConsumerService.java
@@ -131,12 +131,10 @@ public static class MultiBucketConsumer implements IntConsumer {
         private final int limit;
         private final CircuitBreaker breaker;
 
-        // aggregations execute in a single thread for both sequential
-        // and concurrent search, so no atomic here
+        // count is currently only updated in final reduce phase which is executed in single thread for both concurrent and non-concurrent
+        // search
         private int count;
-
-        // will be updated by multiple threads in concurrent search
-        // hence making it as LongAdder
+        // will be updated by multiple threads in concurrent search hence making it as LongAdder
         private final LongAdder callCount;
         private volatile boolean circuitBreakerTripped;
         private final int availProcessors;
diff --git a/server/src/main/java/org/opensearch/search/aggregations/NonGlobalAggCollectorManager.java b/server/src/main/java/org/opensearch/search/aggregations/NonGlobalAggCollectorManager.java
index 8b0a1530b5505..5408a19c8ca50 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/NonGlobalAggCollectorManager.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/NonGlobalAggCollectorManager.java
@@ -12,8 +12,10 @@
 import org.apache.lucene.search.CollectorManager;
 import org.opensearch.search.internal.SearchContext;
 import org.opensearch.search.profile.query.CollectorResult;
+import org.opensearch.search.query.ReduceableSearchResult;
 
 import java.io.IOException;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Objects;
 
@@ -42,6 +44,19 @@ public Collector newCollector() throws IOException {
         }
     }
 
+    @Override
+    public ReduceableSearchResult reduce(Collection<Collector> collectors) throws IOException {
+        // If there are no leaves then in concurrent search case postCollection, and subsequently buildAggregation, will not be called in
+        // search path. Since we build the InternalAggregation in postCollection that will not get created in such cases either. Therefore
+        // we need to manually processPostCollection here to build empty InternalAggregation objects for this collector tree.
+        if (context.searcher().getLeafContexts().isEmpty()) {
+            for (Collector c : collectors) {
+                context.bucketCollectorProcessor().processPostCollection(c);
+            }
+        }
+        return super.reduce(collectors);
+    }
+
     @Override
     protected AggregationReduceableSearchResult buildAggregationResult(InternalAggregations internalAggregations) {
         // Reduce the aggregations across slices before sending to the coordinator. We will perform shard level reduce as long as any slices
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/FastFilterRewriteHelper.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/FastFilterRewriteHelper.java
index e587b7f169e5f..c8ce39a52f869 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/FastFilterRewriteHelper.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/FastFilterRewriteHelper.java
@@ -10,12 +10,13 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
-import org.apache.lucene.document.LongPoint;
 import org.apache.lucene.index.DocValues;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.NumericDocValues;
 import org.apache.lucene.index.PointValues;
+import org.apache.lucene.search.CollectionTerminatedException;
 import org.apache.lucene.search.ConstantScoreQuery;
+import org.apache.lucene.search.DocIdSetIterator;
 import org.apache.lucene.search.FieldExistsQuery;
 import org.apache.lucene.search.IndexOrDocValuesQuery;
 import org.apache.lucene.search.MatchAllDocsQuery;
@@ -23,24 +24,30 @@
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.ScoreMode;
 import org.apache.lucene.search.Weight;
+import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.NumericUtils;
+import org.opensearch.common.CheckedRunnable;
 import org.opensearch.common.Rounding;
 import org.opensearch.common.lucene.search.function.FunctionScoreQuery;
 import org.opensearch.index.mapper.DateFieldMapper;
 import org.opensearch.index.mapper.DocCountFieldMapper;
 import org.opensearch.index.mapper.MappedFieldType;
 import org.opensearch.index.query.DateRangeIncludingNowQuery;
+import org.opensearch.search.aggregations.bucket.composite.CompositeAggregator;
 import org.opensearch.search.aggregations.bucket.composite.CompositeValuesSourceConfig;
 import org.opensearch.search.aggregations.bucket.composite.RoundingValuesSource;
 import org.opensearch.search.aggregations.bucket.histogram.LongBounds;
 import org.opensearch.search.internal.SearchContext;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.OptionalLong;
 import java.util.function.BiConsumer;
+import java.util.function.BiFunction;
 import java.util.function.Function;
 
 import static org.apache.lucene.search.DocIdSetIterator.NO_MORE_DOCS;
@@ -63,7 +70,6 @@ private FastFilterRewriteHelper() {}
 
     private static final Logger logger = LogManager.getLogger(FastFilterRewriteHelper.class);
 
-    private static final int MAX_NUM_FILTER_BUCKETS = 1024;
     private static final Map<Class<?>, Function<Query, Query>> queryWrappers;
 
     // Initialize the wrapper map for unwrapping the query
@@ -129,9 +135,10 @@ private static long[] getSegmentBounds(final LeafReaderContext context, final St
     }
 
     /**
-     * This method also acts as a pre-condition check for the optimization
+     * Gets the min and max bounds of the field for the shard search
+     * Depending on the query part, the bounds are computed differently
      *
-     * @return null if the processed query not as expected
+     * @return null if the processed query not supported by the optimization
      */
     public static long[] getDateHistoAggBounds(final SearchContext context, final String fieldName) throws IOException {
         final Query cq = unwrapIntoConcreteQuery(context.query());
@@ -168,66 +175,6 @@ private static long[] getBoundsWithRangeQuery(PointRangeQuery prq, String fieldN
         return null;
     }
 
-    /**
-     * Creates the date range filters for aggregations using the interval, min/max
-     * bounds and prepared rounding
-     */
-    private static Weight[] createFilterForAggregations(
-        final SearchContext context,
-        final DateFieldMapper.DateFieldType fieldType,
-        final long interval,
-        final Rounding.Prepared preparedRounding,
-        long low,
-        final long high
-    ) throws IOException {
-        // Calculate the number of buckets using range and interval
-        long roundedLow = preparedRounding.round(fieldType.convertNanosToMillis(low));
-        long prevRounded = roundedLow;
-        int bucketCount = 0;
-        while (roundedLow <= fieldType.convertNanosToMillis(high)) {
-            bucketCount++;
-            if (bucketCount > MAX_NUM_FILTER_BUCKETS) {
-                logger.debug("Max number of filters reached [{}], skip the fast filter optimization", MAX_NUM_FILTER_BUCKETS);
-                return null;
-            }
-            // Below rounding is needed as the interval could return in
-            // non-rounded values for something like calendar month
-            roundedLow = preparedRounding.round(roundedLow + interval);
-            if (prevRounded == roundedLow) break; // prevents getting into an infinite loop
-            prevRounded = roundedLow;
-        }
-
-        Weight[] filters = null;
-        if (bucketCount > 0) {
-            filters = new Weight[bucketCount];
-            roundedLow = preparedRounding.round(fieldType.convertNanosToMillis(low));
-
-            int i = 0;
-            while (i < bucketCount) {
-                // Calculate the lower bucket bound
-                final byte[] lower = new byte[8];
-                NumericUtils.longToSortableBytes(i == 0 ? low : fieldType.convertRoundedMillisToNanos(roundedLow), lower, 0);
-
-                // Calculate the upper bucket bound
-                roundedLow = preparedRounding.round(roundedLow + interval);
-                final byte[] upper = new byte[8];
-                NumericUtils.longToSortableBytes(i + 1 == bucketCount ? high :
-                // Subtract -1 if the minimum is roundedLow as roundedLow itself
-                // is included in the next bucket
-                    fieldType.convertRoundedMillisToNanos(roundedLow) - 1, upper, 0);
-
-                filters[i++] = context.searcher().createWeight(new PointRangeQuery(fieldType.name(), lower, upper, 1) {
-                    @Override
-                    protected String toString(int dimension, byte[] value) {
-                        return Long.toString(LongPoint.decodeDimension(value, 0));
-                    }
-                }, ScoreMode.COMPLETE_NO_SCORES, 1);
-            }
-        }
-
-        return filters;
-    }
-
     /**
      * Context object for fast filter optimization
      * <p>
@@ -235,12 +182,24 @@ protected String toString(int dimension, byte[] value) {
      */
     public static class FastFilterContext {
         private boolean rewriteable = false;
-        private Weight[] filters = null;
-        private boolean filtersBuiltAtShardLevel = false;
+        private boolean rangesBuiltAtShardLevel = false;
 
         private AggregationType aggregationType;
         private final SearchContext context;
 
+        private String fieldName;
+        private long[][] ranges;
+
+        // debug info related fields
+        public int leaf;
+        public int inner;
+        public int segments;
+        public int optimizedSegments;
+
+        public void setFieldName(String fieldName) {
+            this.fieldName = fieldName;
+        }
+
         public FastFilterContext(SearchContext context) {
             this.context = context;
         }
@@ -254,30 +213,34 @@ public void setAggregationType(AggregationType aggregationType) {
         }
 
         public boolean isRewriteable(final Object parent, final int subAggLength) {
+            if (context.maxAggRewriteFilters() == 0) return false;
+
             boolean rewriteable = aggregationType.isRewriteable(parent, subAggLength);
             logger.debug("Fast filter rewriteable: {} for shard {}", rewriteable, context.indexShard().shardId());
             this.rewriteable = rewriteable;
             return rewriteable;
         }
 
-        public void buildFastFilter() throws IOException {
-            assert filters == null : "Filters should only be built once, but they are already built";
-            this.filters = this.aggregationType.buildFastFilter(context);
-            if (filters != null) {
-                logger.debug("Fast filter built for shard {}", context.indexShard().shardId());
-                filtersBuiltAtShardLevel = true;
+        public void buildRanges() throws IOException {
+            assert ranges == null : "Ranges should only be built once at shard level, but they are already built";
+            this.ranges = this.aggregationType.buildRanges(context);
+            if (ranges != null) {
+                logger.debug("Ranges built for shard {}", context.indexShard().shardId());
+                rangesBuiltAtShardLevel = true;
             }
         }
 
-        /**
-         * Built filters for a segment
-         */
-        public Weight[] buildFastFilter(LeafReaderContext leaf) throws IOException {
-            Weight[] filters = this.aggregationType.buildFastFilter(leaf, context);
-            if (filters != null) {
-                logger.debug("Fast filter built for shard {} segment {}", context.indexShard().shardId(), leaf.ord);
+        public long[][] buildRanges(LeafReaderContext leaf) throws IOException {
+            long[][] ranges = this.aggregationType.buildRanges(leaf, context);
+            if (ranges != null) {
+                logger.debug("Ranges built for shard {} segment {}", context.indexShard().shardId(), leaf.ord);
             }
-            return filters;
+            return ranges;
+        }
+
+        private void consumeDebugInfo(DebugInfo debug) {
+            leaf += debug.leaf;
+            inner += debug.inner;
         }
     }
 
@@ -285,16 +248,11 @@ public Weight[] buildFastFilter(LeafReaderContext leaf) throws IOException {
      * Different types have different pre-conditions, filter building logic, etc.
      */
     interface AggregationType {
-
         boolean isRewriteable(Object parent, int subAggLength);
 
-        Weight[] buildFastFilter(SearchContext ctx) throws IOException;
+        long[][] buildRanges(SearchContext ctx) throws IOException;
 
-        Weight[] buildFastFilter(LeafReaderContext leaf, SearchContext ctx) throws IOException;
-
-        default int getSize() {
-            return Integer.MAX_VALUE;
-        }
+        long[][] buildRanges(LeafReaderContext leaf, SearchContext ctx) throws IOException;
     }
 
     /**
@@ -328,20 +286,13 @@ public boolean isRewriteable(Object parent, int subAggLength) {
         }
 
         @Override
-        public Weight[] buildFastFilter(SearchContext context) throws IOException {
+        public long[][] buildRanges(SearchContext context) throws IOException {
             long[] bounds = getDateHistoAggBounds(context, fieldType.name());
             logger.debug("Bounds are {} for shard {}", bounds, context.indexShard().shardId());
-            return buildFastFilter(context, bounds);
-        }
-
-        @Override
-        public Weight[] buildFastFilter(LeafReaderContext leaf, SearchContext context) throws IOException {
-            long[] bounds = getSegmentBounds(leaf, fieldType.name());
-            logger.debug("Bounds are {} for shard {} segment {}", bounds, context.indexShard().shardId(), leaf.ord);
-            return buildFastFilter(context, bounds);
+            return buildRanges(context, bounds);
         }
 
-        private Weight[] buildFastFilter(SearchContext context, long[] bounds) throws IOException {
+        private long[][] buildRanges(SearchContext context, long[] bounds) throws IOException {
             bounds = processHardBounds(bounds);
             if (bounds == null) {
                 return null;
@@ -358,7 +309,7 @@ private Weight[] buildFastFilter(SearchContext context, long[] bounds) throws IO
             // process the after key of composite agg
             processAfterKey(bounds, interval);
 
-            return FastFilterRewriteHelper.createFilterForAggregations(
+            return FastFilterRewriteHelper.createRangesFromAgg(
                 context,
                 (DateFieldMapper.DateFieldType) fieldType,
                 interval,
@@ -368,6 +319,13 @@ private Weight[] buildFastFilter(SearchContext context, long[] bounds) throws IO
             );
         }
 
+        @Override
+        public long[][] buildRanges(LeafReaderContext leaf, SearchContext context) throws IOException {
+            long[] bounds = getSegmentBounds(leaf, fieldType.name());
+            logger.debug("Bounds are {} for shard {} segment {}", bounds, context.indexShard().shardId(), leaf.ord);
+            return buildRanges(context, bounds);
+        }
+
         protected abstract Rounding getRounding(final long low, final long high);
 
         protected abstract Rounding.Prepared getRoundingPrepared();
@@ -411,7 +369,7 @@ public static long getBucketOrd(long bucketOrd) {
     }
 
     /**
-     * Try to get the bucket doc counts from the fast filters for the aggregation
+     * Try to get the bucket doc counts for the date histogram aggregation
      * <p>
      * Usage: invoked at segment level — in getLeafCollector of aggregator
      *
@@ -422,11 +380,18 @@ public static boolean tryFastFilterAggregation(
         FastFilterContext fastFilterContext,
         final BiConsumer<Long, Integer> incrementDocCount
     ) throws IOException {
-        if (fastFilterContext == null) return false;
+        fastFilterContext.segments++;
         if (!fastFilterContext.rewriteable) {
             return false;
         }
 
+        if (ctx.reader().hasDeletions()) return false;
+
+        PointValues values = ctx.reader().getPointValues(fastFilterContext.fieldName);
+        if (values == null) return false;
+        // only proceed if every document corresponds to exactly one point
+        if (values.getDocCount() != values.size()) return false;
+
         NumericDocValues docCountValues = DocValues.getNumeric(ctx.reader(), DocCountFieldMapper.NAME);
         if (docCountValues.nextDoc() != NO_MORE_DOCS) {
             logger.debug(
@@ -437,61 +402,330 @@ public static boolean tryFastFilterAggregation(
             return false;
         }
 
-        // if no filters built at shard level (see getDateHistoAggBounds method for possible reasons)
-        // check if the query is functionally match-all at segment level
-        if (!fastFilterContext.filtersBuiltAtShardLevel && !segmentMatchAll(fastFilterContext.context, ctx)) {
+        // even if no ranges built at shard level, we can still perform the optimization
+        // when functionally match-all at segment level
+        if (!fastFilterContext.rangesBuiltAtShardLevel && !segmentMatchAll(fastFilterContext.context, ctx)) {
             return false;
         }
-        Weight[] filters = fastFilterContext.filters;
-        if (filters == null) {
+        long[][] ranges = fastFilterContext.ranges;
+        if (ranges == null) {
             logger.debug(
                 "Shard {} segment {} functionally match all documents. Build the fast filter",
                 fastFilterContext.context.indexShard().shardId(),
                 ctx.ord
             );
-            filters = fastFilterContext.buildFastFilter(ctx);
-            if (filters == null) {
+            ranges = fastFilterContext.buildRanges(ctx);
+            if (ranges == null) {
                 return false;
             }
         }
 
-        final int[] counts = new int[filters.length];
-        int i;
-        for (i = 0; i < filters.length; i++) {
-            counts[i] = filters[i].count(ctx);
-            if (counts[i] == -1) {
-                // Cannot use the optimization if any of the counts
-                // is -1 indicating the segment might have deleted documents
-                return false;
+        final AggregationType aggregationType = fastFilterContext.aggregationType;
+        assert aggregationType instanceof AbstractDateHistogramAggregationType;
+        final DateFieldMapper.DateFieldType fieldType = ((AbstractDateHistogramAggregationType) aggregationType).getFieldType();
+        int size = Integer.MAX_VALUE;
+        if (aggregationType instanceof CompositeAggregator.CompositeAggregationType) {
+            size = ((CompositeAggregator.CompositeAggregationType) aggregationType).getSize();
+        }
+        DebugInfo debugInfo = multiRangesTraverse(values.getPointTree(), ranges, incrementDocCount, fieldType, size);
+        fastFilterContext.consumeDebugInfo(debugInfo);
+
+        fastFilterContext.optimizedSegments++;
+        logger.debug("Fast filter optimization applied to shard {} segment {}", fastFilterContext.context.indexShard().shardId(), ctx.ord);
+        logger.debug("crossed leaf nodes: {}, inner nodes: {}", fastFilterContext.leaf, fastFilterContext.inner);
+        return true;
+    }
+
+    private static boolean segmentMatchAll(SearchContext ctx, LeafReaderContext leafCtx) throws IOException {
+        Weight weight = ctx.searcher().createWeight(ctx.query(), ScoreMode.COMPLETE_NO_SCORES, 1f);
+        return weight != null && weight.count(leafCtx) == leafCtx.reader().numDocs();
+    }
+
+    /**
+     * Creates the date ranges from date histo aggregations using its interval,
+     * and min/max boundaries
+     */
+    private static long[][] createRangesFromAgg(
+        final SearchContext context,
+        final DateFieldMapper.DateFieldType fieldType,
+        final long interval,
+        final Rounding.Prepared preparedRounding,
+        long low,
+        final long high
+    ) {
+        // Calculate the number of buckets using range and interval
+        long roundedLow = preparedRounding.round(fieldType.convertNanosToMillis(low));
+        long prevRounded = roundedLow;
+        int bucketCount = 0;
+        while (roundedLow <= fieldType.convertNanosToMillis(high)) {
+            bucketCount++;
+            int maxNumFilterBuckets = context.maxAggRewriteFilters();
+            if (bucketCount > maxNumFilterBuckets) {
+                logger.debug("Max number of filters reached [{}], skip the fast filter optimization", maxNumFilterBuckets);
+                return null;
+            }
+            // Below rounding is needed as the interval could return in
+            // non-rounded values for something like calendar month
+            roundedLow = preparedRounding.round(roundedLow + interval);
+            if (prevRounded == roundedLow) break; // prevents getting into an infinite loop
+            prevRounded = roundedLow;
+        }
+
+        long[][] ranges = new long[bucketCount][2];
+        if (bucketCount > 0) {
+            roundedLow = preparedRounding.round(fieldType.convertNanosToMillis(low));
+
+            int i = 0;
+            while (i < bucketCount) {
+                // Calculate the lower bucket bound
+                long lower = i == 0 ? low : fieldType.convertRoundedMillisToNanos(roundedLow);
+                roundedLow = preparedRounding.round(roundedLow + interval);
+
+                // Subtract -1 if the minimum is roundedLow as roundedLow itself
+                // is included in the next bucket
+                long upper = i + 1 == bucketCount ? high : fieldType.convertRoundedMillisToNanos(roundedLow) - 1;
+
+                ranges[i][0] = lower;
+                ranges[i][1] = upper;
+                i++;
             }
         }
 
-        int s = 0;
-        int size = fastFilterContext.aggregationType.getSize();
-        for (i = 0; i < filters.length; i++) {
-            if (counts[i] > 0) {
-                long bucketKey = i; // the index of filters is the key for filters aggregation
-                if (fastFilterContext.aggregationType instanceof AbstractDateHistogramAggregationType) {
-                    final DateFieldMapper.DateFieldType fieldType =
-                        ((AbstractDateHistogramAggregationType) fastFilterContext.aggregationType).getFieldType();
-                    bucketKey = fieldType.convertNanosToMillis(
-                        NumericUtils.sortableBytesToLong(((PointRangeQuery) filters[i].getQuery()).getLowerPoint(), 0)
-                    );
+        return ranges;
+    }
+
+    /**
+     * @param maxNumNonZeroRanges the number of non-zero ranges to collect
+     */
+    private static DebugInfo multiRangesTraverse(
+        final PointValues.PointTree tree,
+        final long[][] ranges,
+        final BiConsumer<Long, Integer> incrementDocCount,
+        final DateFieldMapper.DateFieldType fieldType,
+        final int maxNumNonZeroRanges
+    ) throws IOException {
+        // ranges are connected and in ascending order
+        Iterator<long[]> rangeIter = Arrays.stream(ranges).iterator();
+        long[] activeRange = rangeIter.next();
+
+        // make sure the first range at least crosses the min value of the tree
+        DebugInfo debugInfo = new DebugInfo();
+        if (activeRange[0] > NumericUtils.sortableBytesToLong(tree.getMaxPackedValue(), 0)) {
+            logger.debug("No ranges match the query, skip the fast filter optimization");
+            return debugInfo;
+        }
+        while (activeRange[1] < NumericUtils.sortableBytesToLong(tree.getMinPackedValue(), 0)) {
+            if (!rangeIter.hasNext()) {
+                logger.debug("No ranges match the query, skip the fast filter optimization");
+                return debugInfo;
+            }
+            activeRange = rangeIter.next();
+        }
+
+        RangeCollectorForPointTree collector = new RangeCollectorForPointTree(
+            incrementDocCount,
+            fieldType,
+            rangeIter,
+            maxNumNonZeroRanges,
+            activeRange
+        );
+
+        final ArrayUtil.ByteArrayComparator comparator = ArrayUtil.getUnsignedComparator(8);
+        PointValues.IntersectVisitor visitor = getIntersectVisitor(collector, comparator);
+        try {
+            intersectWithRanges(visitor, tree, collector, debugInfo);
+        } catch (CollectionTerminatedException e) {
+            logger.debug("Early terminate since no more range to collect");
+        }
+        collector.finalizePreviousRange();
+
+        return debugInfo;
+    }
+
+    private static void intersectWithRanges(
+        PointValues.IntersectVisitor visitor,
+        PointValues.PointTree pointTree,
+        RangeCollectorForPointTree collector,
+        DebugInfo debug
+    ) throws IOException {
+        PointValues.Relation r = visitor.compare(pointTree.getMinPackedValue(), pointTree.getMaxPackedValue());
+
+        switch (r) {
+            case CELL_INSIDE_QUERY:
+                collector.countNode((int) pointTree.size());
+                debug.visitInner();
+                break;
+            case CELL_CROSSES_QUERY:
+                if (pointTree.moveToChild()) {
+                    do {
+                        intersectWithRanges(visitor, pointTree, collector, debug);
+                    } while (pointTree.moveToSibling());
+                    pointTree.moveToParent();
+                } else {
+                    pointTree.visitDocValues(visitor);
+                    debug.visitLeaf();
+                }
+                break;
+            case CELL_OUTSIDE_QUERY:
+        }
+    }
+
+    private static PointValues.IntersectVisitor getIntersectVisitor(
+        RangeCollectorForPointTree collector,
+        ArrayUtil.ByteArrayComparator comparator
+    ) {
+        return new PointValues.IntersectVisitor() {
+            @Override
+            public void visit(int docID) throws IOException {
+                // this branch should be unreachable
+                throw new UnsupportedOperationException(
+                    "This IntersectVisitor does not perform any actions on a " + "docID=" + docID + " node being visited"
+                );
+            }
+
+            @Override
+            public void visit(int docID, byte[] packedValue) throws IOException {
+                visitPoints(packedValue, collector::count);
+            }
+
+            @Override
+            public void visit(DocIdSetIterator iterator, byte[] packedValue) throws IOException {
+                visitPoints(packedValue, () -> {
+                    for (int doc = iterator.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = iterator.nextDoc()) {
+                        collector.count();
+                    }
+                });
+            }
+
+            private void visitPoints(byte[] packedValue, CheckedRunnable<IOException> collect) throws IOException {
+                if (comparator.compare(packedValue, 0, collector.activeRangeAsByteArray[1], 0) > 0) {
+                    // need to move to next range
+                    collector.finalizePreviousRange();
+                    if (collector.iterateRangeEnd(packedValue, this::compareByteValue)) {
+                        throw new CollectionTerminatedException();
+                    }
+                }
+
+                if (pointCompare(collector.activeRangeAsByteArray[0], collector.activeRangeAsByteArray[1], packedValue)) {
+                    collect.run();
+                }
+            }
+
+            private boolean pointCompare(byte[] lower, byte[] upper, byte[] packedValue) {
+                if (compareByteValue(packedValue, lower) < 0) {
+                    return false;
+                }
+                return compareByteValue(packedValue, upper) <= 0;
+            }
+
+            private int compareByteValue(byte[] value1, byte[] value2) {
+                return comparator.compare(value1, 0, value2, 0);
+            }
+
+            @Override
+            public PointValues.Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
+                byte[] rangeMin = collector.activeRangeAsByteArray[0];
+                byte[] rangeMax = collector.activeRangeAsByteArray[1];
+
+                if (compareByteValue(rangeMax, minPackedValue) < 0) {
+                    collector.finalizePreviousRange();
+                    if (collector.iterateRangeEnd(minPackedValue, this::compareByteValue)) {
+                        throw new CollectionTerminatedException();
+                    }
+                    // compare the next range with this node's min max again
+                    // new rangeMin = previous rangeMax + 1 <= min
+                    rangeMax = collector.activeRangeAsByteArray[1];
                 }
-                incrementDocCount.accept(bucketKey, counts[i]);
-                s++;
-                if (s > size) {
-                    break;
+
+                if (compareByteValue(rangeMin, minPackedValue) > 0 || compareByteValue(rangeMax, maxPackedValue) < 0) {
+                    return PointValues.Relation.CELL_CROSSES_QUERY;
+                } else {
+                    return PointValues.Relation.CELL_INSIDE_QUERY;
                 }
             }
+        };
+    }
+
+    private static class RangeCollectorForPointTree {
+        private final BiConsumer<Long, Integer> incrementDocCount;
+        private final DateFieldMapper.DateFieldType fieldType;
+        private int counter = 0;
+
+        private long[] activeRange;
+        private byte[][] activeRangeAsByteArray;
+        private final Iterator<long[]> rangeIter;
+
+        private int visitedRange = 0;
+        private final int maxNumNonZeroRange;
+
+        public RangeCollectorForPointTree(
+            BiConsumer<Long, Integer> incrementDocCount,
+            DateFieldMapper.DateFieldType fieldType,
+            Iterator<long[]> rangeIter,
+            int maxNumNonZeroRange,
+            long[] activeRange
+        ) {
+            this.incrementDocCount = incrementDocCount;
+            this.fieldType = fieldType;
+            this.rangeIter = rangeIter;
+            this.maxNumNonZeroRange = maxNumNonZeroRange;
+            this.activeRange = activeRange;
+            this.activeRangeAsByteArray = activeRangeAsByteArray();
         }
 
-        logger.debug("Fast filter optimization applied to shard {} segment {}", fastFilterContext.context.indexShard().shardId(), ctx.ord);
-        return true;
+        private void count() {
+            counter++;
+        }
+
+        private void countNode(int count) {
+            counter += count;
+        }
+
+        private void finalizePreviousRange() {
+            if (counter > 0) {
+                logger.debug("finalize previous range: {}", activeRange[0]);
+                logger.debug("counter: {}", counter);
+                incrementDocCount.accept(fieldType.convertNanosToMillis(activeRange[0]), counter);
+                counter = 0;
+            }
+        }
+
+        /**
+         * @return true when iterator exhausted or collect enough non-zero ranges
+         */
+        private boolean iterateRangeEnd(byte[] value, BiFunction<byte[], byte[], Integer> comparator) {
+            // the new value may not be contiguous to the previous one
+            // so try to find the first next range that cross the new value
+            while (comparator.apply(activeRangeAsByteArray[1], value) < 0) {
+                if (!rangeIter.hasNext()) {
+                    return true;
+                }
+                activeRange = rangeIter.next();
+                activeRangeAsByteArray = activeRangeAsByteArray();
+            }
+            visitedRange++;
+            return visitedRange > maxNumNonZeroRange;
+        }
+
+        private byte[][] activeRangeAsByteArray() {
+            byte[] lower = new byte[8];
+            byte[] upper = new byte[8];
+            NumericUtils.longToSortableBytes(activeRange[0], lower, 0);
+            NumericUtils.longToSortableBytes(activeRange[1], upper, 0);
+            return new byte[][] { lower, upper };
+        }
     }
 
-    private static boolean segmentMatchAll(SearchContext ctx, LeafReaderContext leafCtx) throws IOException {
-        Weight weight = ctx.searcher().createWeight(ctx.query(), ScoreMode.COMPLETE_NO_SCORES, 1f);
-        return weight != null && weight.count(leafCtx) == leafCtx.reader().numDocs();
+    private static class DebugInfo {
+        private int leaf = 0; // leaf node visited
+        private int inner = 0; // inner node visited
+
+        private void visitLeaf() {
+            leaf++;
+        }
+
+        private void visitInner() {
+            inner++;
+        }
     }
 }
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregationFactory.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregationFactory.java
index 4af14ab014db5..6c5619a843fae 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregationFactory.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregationFactory.java
@@ -40,6 +40,7 @@
 import org.opensearch.search.internal.SearchContext;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.Map;
 
 /**
@@ -80,7 +81,7 @@ protected Aggregator createInternal(
 
     @Override
     protected boolean supportsConcurrentSegmentSearch() {
-        // See https://github.com/opensearch-project/OpenSearch/issues/12331 for details
-        return false;
+        // Disable concurrent search if any scripting is used. See https://github.com/opensearch-project/OpenSearch/issues/12331 for details
+        return Arrays.stream(sources).noneMatch(CompositeValuesSourceConfig::hasScript);
     }
 }
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregator.java
index b97c814cdf645..3713d8f83990d 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregator.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregator.java
@@ -87,6 +87,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.function.BiConsumer;
 import java.util.function.LongUnaryOperator;
 import java.util.stream.Collectors;
 
@@ -97,7 +98,7 @@
  *
  * @opensearch.internal
  */
-final class CompositeAggregator extends BucketsAggregator {
+public final class CompositeAggregator extends BucketsAggregator {
     private final int size;
     private final List<String> sourceNames;
     private final int[] reverseMuls;
@@ -171,14 +172,15 @@ final class CompositeAggregator extends BucketsAggregator {
             // bucketOrds is used for saving date histogram results
             bucketOrds = LongKeyedBucketOrds.build(context.bigArrays(), CardinalityUpperBound.ONE);
             preparedRounding = ((CompositeAggregationType) fastFilterContext.getAggregationType()).getRoundingPrepared();
-            fastFilterContext.buildFastFilter();
+            fastFilterContext.setFieldName(sourceConfigs[0].fieldType().name());
+            fastFilterContext.buildRanges();
         }
     }
 
     /**
      * Currently the filter rewrite is only supported for date histograms
      */
-    private class CompositeAggregationType extends FastFilterRewriteHelper.AbstractDateHistogramAggregationType {
+    public class CompositeAggregationType extends FastFilterRewriteHelper.AbstractDateHistogramAggregationType {
         private final RoundingValuesSource valuesSource;
         private long afterKey = -1L;
 
@@ -210,7 +212,6 @@ protected void processAfterKey(long[] bound, long interval) {
             }
         }
 
-        @Override
         public int getSize() {
             return size;
         }
@@ -706,4 +707,14 @@ private static class Entry {
             this.docIdSet = docIdSet;
         }
     }
+
+    @Override
+    public void collectDebugInfo(BiConsumer<String, Object> add) {
+        if (fastFilterContext.optimizedSegments > 0) {
+            add.accept("optimized_segments", fastFilterContext.optimizedSegments);
+            add.accept("unoptimized_segments", fastFilterContext.segments - fastFilterContext.optimizedSegments);
+            add.accept("leaf_visited", fastFilterContext.leaf);
+            add.accept("inner_visited", fastFilterContext.inner);
+        }
+    }
 }
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregator.java
index 12aefc540e75c..f326426800909 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregator.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/AutoDateHistogramAggregator.java
@@ -166,7 +166,8 @@ private AutoDateHistogramAggregator(
             )
         );
         if (fastFilterContext.isRewriteable(parent, subAggregators.length)) {
-            fastFilterContext.buildFastFilter();
+            fastFilterContext.setFieldName(valuesSourceConfig.fieldType().name());
+            fastFilterContext.buildRanges();
         }
     }
 
@@ -307,6 +308,17 @@ protected final void merge(long[] mergeMap, long newNumBuckets) {
         }
     }
 
+    @Override
+    public void collectDebugInfo(BiConsumer<String, Object> add) {
+        super.collectDebugInfo(add);
+        if (fastFilterContext.optimizedSegments > 0) {
+            add.accept("optimized_segments", fastFilterContext.optimizedSegments);
+            add.accept("unoptimized_segments", fastFilterContext.segments - fastFilterContext.optimizedSegments);
+            add.accept("leaf_visited", fastFilterContext.leaf);
+            add.accept("inner_visited", fastFilterContext.inner);
+        }
+    }
+
     /**
      * Initially it uses the most fine grained rounding configuration possible
      * but as more data arrives it rebuckets the data until it "fits" in the
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java
index 0e830106c8284..dd4ee9196fd62 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramAggregator.java
@@ -126,7 +126,8 @@ class DateHistogramAggregator extends BucketsAggregator implements SizedBucketAg
             )
         );
         if (fastFilterContext.isRewriteable(parent, subAggregators.length)) {
-            fastFilterContext.buildFastFilter();
+            fastFilterContext.setFieldName(valuesSourceConfig.fieldType().name());
+            fastFilterContext.buildRanges();
         }
     }
 
@@ -255,6 +256,12 @@ public void doClose() {
     @Override
     public void collectDebugInfo(BiConsumer<String, Object> add) {
         add.accept("total_buckets", bucketOrds.size());
+        if (fastFilterContext.optimizedSegments > 0) {
+            add.accept("optimized_segments", fastFilterContext.optimizedSegments);
+            add.accept("unoptimized_segments", fastFilterContext.segments - fastFilterContext.optimizedSegments);
+            add.accept("leaf_visited", fastFilterContext.leaf);
+            add.accept("inner_visited", fastFilterContext.inner);
+        }
     }
 
     /**
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java
index 69fda2f3f6133..9e40f7b4c9b3e 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java
@@ -45,6 +45,7 @@
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.PriorityQueue;
+import org.opensearch.common.SetOnce;
 import org.opensearch.common.lease.Releasable;
 import org.opensearch.common.lease.Releasables;
 import org.opensearch.common.util.LongArray;
@@ -94,8 +95,8 @@ public class GlobalOrdinalsStringTermsAggregator extends AbstractStringTermsAggr
     private final long valueCount;
     private final String fieldName;
     private Weight weight;
-    private final GlobalOrdLookupFunction lookupGlobalOrd;
     protected final CollectionStrategy collectionStrategy;
+    private final SetOnce<SortedSetDocValues> dvs = new SetOnce<>();
     protected int segmentsWithSingleValuedOrds = 0;
     protected int segmentsWithMultiValuedOrds = 0;
 
@@ -129,11 +130,10 @@ public GlobalOrdinalsStringTermsAggregator(
         this.resultStrategy = resultStrategy.apply(this); // ResultStrategy needs a reference to the Aggregator to do its job.
         this.valuesSource = valuesSource;
         final IndexReader reader = context.searcher().getIndexReader();
-        final SortedSetDocValues values = reader.leaves().size() > 0
+        final SortedSetDocValues values = !reader.leaves().isEmpty()
             ? valuesSource.globalOrdinalsValues(context.searcher().getIndexReader().leaves().get(0))
             : DocValues.emptySortedSet();
         this.valueCount = values.getValueCount();
-        this.lookupGlobalOrd = values::lookupOrd;
         this.acceptedGlobalOrdinals = includeExclude == null ? ALWAYS_TRUE : includeExclude.acceptedGlobalOrdinals(values)::get;
         if (remapGlobalOrds) {
             this.collectionStrategy = new RemapGlobalOrds(cardinality);
@@ -885,7 +885,10 @@ PriorityQueue<OrdBucket> buildPriorityQueue(int size) {
         }
 
         StringTerms.Bucket convertTempBucketToRealBucket(OrdBucket temp) throws IOException {
-            BytesRef term = BytesRef.deepCopyOf(lookupGlobalOrd.apply(temp.globalOrd));
+            // Recreate DocValues as needed for concurrent segment search
+            SortedSetDocValues values = getDocValues();
+            BytesRef term = BytesRef.deepCopyOf(values.lookupOrd(temp.globalOrd));
+
             StringTerms.Bucket result = new StringTerms.Bucket(term, temp.docCount, null, showTermDocCountError, 0, format);
             result.bucketOrd = temp.bucketOrd;
             result.docCountError = 0;
@@ -1001,7 +1004,9 @@ BucketUpdater<SignificantStringTerms.Bucket> bucketUpdater(long owningBucketOrd)
             long subsetSize = subsetSize(owningBucketOrd);
             return (spare, globalOrd, bucketOrd, docCount) -> {
                 spare.bucketOrd = bucketOrd;
-                oversizedCopy(lookupGlobalOrd.apply(globalOrd), spare.termBytes);
+                // Recreate DocValues as needed for concurrent segment search
+                SortedSetDocValues values = getDocValues();
+                oversizedCopy(values.lookupOrd(globalOrd), spare.termBytes);
                 spare.subsetDf = docCount;
                 spare.subsetSize = subsetSize;
                 spare.supersetDf = backgroundFrequencies.freq(spare.termBytes);
@@ -1086,4 +1091,18 @@ private void oversizedCopy(BytesRef from, BytesRef to) {
      * Predicate used for {@link #acceptedGlobalOrdinals} if there is no filter.
      */
     private static final LongPredicate ALWAYS_TRUE = l -> true;
+
+    /**
+     * If DocValues have not been initialized yet for reduce phase, create and set them.
+     */
+    private SortedSetDocValues getDocValues() throws IOException {
+        if (dvs.get() == null) {
+            dvs.set(
+                !context.searcher().getIndexReader().leaves().isEmpty()
+                    ? valuesSource.globalOrdinalsValues(context.searcher().getIndexReader().leaves().get(0))
+                    : DocValues.emptySortedSet()
+            );
+        }
+        return dvs.get();
+    }
 }
diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/MultiTermsAggregationBuilder.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/MultiTermsAggregationBuilder.java
index c711b140be192..fb588a04cf93c 100644
--- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/MultiTermsAggregationBuilder.java
+++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/MultiTermsAggregationBuilder.java
@@ -51,10 +51,15 @@
  *           },{
  *             "field": "host"
  *           }],
- *           "order": {"max-cpu": "desc"}
+ *           "order": [{
+ *             "max-cpu": "desc"
+ *           },{
+ *             "max-memory": "desc"
+ *           }]
  *         },
  *         "aggs": {
- *           "max-cpu": { "max": { "field": "cpu" } }
+ *           "max-cpu": { "max": { "field": "cpu" } },
+ *           "max-memory": { "max": { "field": "memory" } }
  *         }
  *       }
  *     }
@@ -80,6 +85,9 @@
  *           "doc_count": 2,
  *           "max-cpu": {
  *             "value": 90.0
+ *           },
+ *           "max-memory": {
+ *             "value": 80.0
  *           }
  *         },
  *         {
@@ -91,6 +99,9 @@
  *           "doc_count": 2,
  *           "max-cpu": {
  *             "value": 70.0
+ *           },
+ *           "max-memory": {
+ *             "value": 90.0
  *           }
  *         }
  *       ]
diff --git a/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java
index bf7045d43ba67..07248a0719c3a 100644
--- a/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java
+++ b/server/src/main/java/org/opensearch/search/builder/SearchSourceBuilder.java
@@ -51,6 +51,8 @@
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.core.xcontent.XContentHelper;
 import org.opensearch.core.xcontent.XContentParser;
+import org.opensearch.index.mapper.DerivedField;
+import org.opensearch.index.mapper.DerivedFieldMapper;
 import org.opensearch.index.query.QueryBuilder;
 import org.opensearch.index.query.QueryRewriteContext;
 import org.opensearch.index.query.Rewriteable;
@@ -113,6 +115,7 @@ public final class SearchSourceBuilder implements Writeable, ToXContentObject, R
     public static final ParseField DOCVALUE_FIELDS_FIELD = new ParseField("docvalue_fields");
     public static final ParseField FETCH_FIELDS_FIELD = new ParseField("fields");
     public static final ParseField SCRIPT_FIELDS_FIELD = new ParseField("script_fields");
+    public static final ParseField DERIVED_FIELDS_FIELD = new ParseField(DerivedFieldMapper.CONTENT_TYPE);
     public static final ParseField SCRIPT_FIELD = new ParseField("script");
     public static final ParseField IGNORE_FAILURE_FIELD = new ParseField("ignore_failure");
     public static final ParseField SORT_FIELD = new ParseField("sort");
@@ -192,6 +195,10 @@ public static HighlightBuilder highlight() {
     private StoredFieldsContext storedFieldsContext;
     private List<FieldAndFormat> docValueFields;
     private List<ScriptField> scriptFields;
+    private Map<String, Object> derivedFieldsObject;
+
+    private List<DerivedField> derivedFields;
+
     private FetchSourceContext fetchSourceContext;
     private List<FieldAndFormat> fetchFields;
 
@@ -282,6 +289,14 @@ public SearchSourceBuilder(StreamInput in) throws IOException {
         if (in.getVersion().onOrAfter(Version.V_2_13_0)) {
             includeNamedQueriesScore = in.readOptionalBoolean();
         }
+        if (in.getVersion().onOrAfter(Version.V_2_14_0)) {
+            if (in.readBoolean()) {
+                derivedFieldsObject = in.readMap();
+            }
+            if (in.readBoolean()) {
+                derivedFields = in.readList(DerivedField::new);
+            }
+        }
     }
 
     @Override
@@ -350,6 +365,18 @@ public void writeTo(StreamOutput out) throws IOException {
         if (out.getVersion().onOrAfter(Version.V_2_13_0)) {
             out.writeOptionalBoolean(includeNamedQueriesScore);
         }
+        if (out.getVersion().onOrAfter(Version.V_2_14_0)) {
+            boolean hasDerivedFieldsObject = derivedFieldsObject != null;
+            out.writeBoolean(hasDerivedFieldsObject);
+            if (hasDerivedFieldsObject) {
+                out.writeMap(derivedFieldsObject);
+            }
+            boolean hasDerivedFields = derivedFields != null;
+            out.writeBoolean(hasDerivedFields);
+            if (hasDerivedFields) {
+                out.writeList(derivedFields);
+            }
+        }
     }
 
     /**
@@ -391,7 +418,7 @@ public QueryBuilder postFilter() {
      */
     public SearchSourceBuilder from(int from) {
         if (from < 0) {
-            throw new IllegalArgumentException("[from] parameter cannot be negative");
+            throw new IllegalArgumentException("[from] parameter cannot be negative, found [" + from + "]");
         }
         this.from = from;
         return this;
@@ -955,6 +982,28 @@ public List<ScriptField> scriptFields() {
         return scriptFields;
     }
 
+    public Map<String, Object> getDerivedFieldsObject() {
+        return derivedFieldsObject;
+    }
+
+    public List<DerivedField> getDerivedFields() {
+        return derivedFields;
+    }
+
+    /**
+     * Adds a derived field with the given name with provided type and script
+     * @param name name of the derived field
+     * @param type type of the derived field
+     * @param script script associated with derived field
+     */
+    public SearchSourceBuilder derivedField(String name, String type, Script script) {
+        if (derivedFields == null) {
+            derivedFields = new ArrayList<>();
+        }
+        derivedFields.add(new DerivedField(name, type, script));
+        return this;
+    }
+
     /**
      * Sets the boost a specific index or alias will receive when the query is executed
      * against it.
@@ -1134,6 +1183,8 @@ private SearchSourceBuilder shallowCopy(
         rewrittenBuilder.seqNoAndPrimaryTerm = seqNoAndPrimaryTerm;
         rewrittenBuilder.collapse = collapse;
         rewrittenBuilder.pointInTimeBuilder = pointInTimeBuilder;
+        rewrittenBuilder.derivedFieldsObject = derivedFieldsObject;
+        rewrittenBuilder.derivedFields = derivedFields;
         return rewrittenBuilder;
     }
 
@@ -1164,9 +1215,9 @@ public void parseXContent(XContentParser parser, boolean checkTrailingTokens) th
                 currentFieldName = parser.currentName();
             } else if (token.isValue()) {
                 if (FROM_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
-                    from = parser.intValue();
+                    from(parser.intValue());
                 } else if (SIZE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
-                    size = parser.intValue();
+                    size(parser.intValue());
                 } else if (TIMEOUT_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
                     timeout = TimeValue.parseTimeValue(parser.text(), null, TIMEOUT_FIELD.getPreferredName());
                 } else if (TERMINATE_AFTER_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
@@ -1281,6 +1332,8 @@ public void parseXContent(XContentParser parser, boolean checkTrailingTokens) th
                         pointInTimeBuilder = PointInTimeBuilder.fromXContent(parser);
                     } else if (SEARCH_PIPELINE.match(currentFieldName, parser.getDeprecationHandler())) {
                         searchPipelineSource = parser.mapOrdered();
+                    } else if (DERIVED_FIELDS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
+                        derivedFieldsObject = parser.map();
                     } else {
                         throw new ParsingException(
                             parser.getTokenLocation(),
@@ -1513,6 +1566,21 @@ public XContentBuilder innerToXContent(XContentBuilder builder, Params params) t
         if (searchPipelineSource != null) {
             builder.field(SEARCH_PIPELINE.getPreferredName(), searchPipelineSource);
         }
+
+        if (derivedFieldsObject != null || derivedFields != null) {
+            builder.startObject(DERIVED_FIELDS_FIELD.getPreferredName());
+            if (derivedFieldsObject != null) {
+                builder.mapContents(derivedFieldsObject);
+            }
+            if (derivedFields != null) {
+                for (DerivedField derivedField : derivedFields) {
+                    derivedField.toXContent(builder, params);
+                }
+            }
+            builder.endObject();
+
+        }
+
         return builder;
     }
 
@@ -1788,7 +1856,9 @@ public int hashCode() {
             extBuilders,
             collapse,
             trackTotalHitsUpTo,
-            pointInTimeBuilder
+            pointInTimeBuilder,
+            derivedFieldsObject,
+            derivedFields
         );
     }
 
@@ -1831,7 +1901,9 @@ public boolean equals(Object obj) {
             && Objects.equals(extBuilders, other.extBuilders)
             && Objects.equals(collapse, other.collapse)
             && Objects.equals(trackTotalHitsUpTo, other.trackTotalHitsUpTo)
-            && Objects.equals(pointInTimeBuilder, other.pointInTimeBuilder);
+            && Objects.equals(pointInTimeBuilder, other.pointInTimeBuilder)
+            && Objects.equals(derivedFieldsObject, other.derivedFieldsObject)
+            && Objects.equals(derivedFields, other.derivedFields);
     }
 
     @Override
diff --git a/server/src/main/java/org/opensearch/search/fetch/FetchContext.java b/server/src/main/java/org/opensearch/search/fetch/FetchContext.java
index 780a6f35524ea..5be3733106655 100644
--- a/server/src/main/java/org/opensearch/search/fetch/FetchContext.java
+++ b/server/src/main/java/org/opensearch/search/fetch/FetchContext.java
@@ -192,10 +192,6 @@ public boolean includeNamedQueriesScore() {
         return searchContext.includeNamedQueriesScore();
     }
 
-    public boolean hasInnerHits() {
-        return searchContext.hasInnerHits();
-    }
-
     /**
      * Configuration for returning inner hits
      */
@@ -217,10 +213,6 @@ public FetchFieldsContext fetchFieldsContext() {
         return searchContext.fetchFieldsContext();
     }
 
-    public boolean hasScriptFields() {
-        return searchContext.hasScriptFields();
-    }
-
     /**
      * Configuration for script fields
      */
diff --git a/server/src/main/java/org/opensearch/search/fetch/subphase/InnerHitsContext.java b/server/src/main/java/org/opensearch/search/fetch/subphase/InnerHitsContext.java
index fa80bb04c77f5..5855a0b3217f3 100644
--- a/server/src/main/java/org/opensearch/search/fetch/subphase/InnerHitsContext.java
+++ b/server/src/main/java/org/opensearch/search/fetch/subphase/InnerHitsContext.java
@@ -119,11 +119,6 @@ public String getName() {
             return name;
         }
 
-        @Override
-        public boolean hasInnerHits() {
-            return childInnerHits != null;
-        }
-
         @Override
         public InnerHitsContext innerHits() {
             return childInnerHits;
diff --git a/server/src/main/java/org/opensearch/search/fetch/subphase/InnerHitsPhase.java b/server/src/main/java/org/opensearch/search/fetch/subphase/InnerHitsPhase.java
index cadad8529da9d..0b07dc35f13bb 100644
--- a/server/src/main/java/org/opensearch/search/fetch/subphase/InnerHitsPhase.java
+++ b/server/src/main/java/org/opensearch/search/fetch/subphase/InnerHitsPhase.java
@@ -64,7 +64,7 @@ public InnerHitsPhase(FetchPhase fetchPhase) {
 
     @Override
     public FetchSubPhaseProcessor getProcessor(FetchContext searchContext) {
-        if (searchContext.hasInnerHits() == false) {
+        if (searchContext.innerHits() == null) {
             return null;
         }
         Map<String, InnerHitsContext.InnerHitSubContext> innerHits = searchContext.innerHits().getInnerHits();
diff --git a/server/src/main/java/org/opensearch/search/fetch/subphase/ScriptFieldsPhase.java b/server/src/main/java/org/opensearch/search/fetch/subphase/ScriptFieldsPhase.java
index bee536dbaf7f6..67d1863050a7b 100644
--- a/server/src/main/java/org/opensearch/search/fetch/subphase/ScriptFieldsPhase.java
+++ b/server/src/main/java/org/opensearch/search/fetch/subphase/ScriptFieldsPhase.java
@@ -54,7 +54,7 @@ public final class ScriptFieldsPhase implements FetchSubPhase {
 
     @Override
     public FetchSubPhaseProcessor getProcessor(FetchContext context) {
-        if (context.hasScriptFields() == false) {
+        if (context.scriptFields() == null) {
             return null;
         }
         List<ScriptFieldsContext.ScriptField> scriptFields = context.scriptFields().fields();
diff --git a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightPhase.java b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightPhase.java
index 2fc9b214e3ebb..b16f06e7e3989 100644
--- a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightPhase.java
+++ b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightPhase.java
@@ -145,6 +145,9 @@ private Map<String, Function<HitContext, FieldHighlightContext>> contextBuilders
             boolean fieldNameContainsWildcards = field.field().contains("*");
             for (String fieldName : fieldNamesToHighlight) {
                 MappedFieldType fieldType = context.mapperService().fieldType(fieldName);
+                if (fieldType == null && context.getQueryShardContext().getDerivedFieldType(fieldName) != null) {
+                    fieldType = context.getQueryShardContext().getDerivedFieldType(fieldName);
+                }
                 if (fieldType == null) {
                     continue;
                 }
@@ -170,12 +173,13 @@ private Map<String, Function<HitContext, FieldHighlightContext>> contextBuilders
                 Query highlightQuery = field.fieldOptions().highlightQuery();
 
                 boolean forceSource = highlightContext.forceSource(field);
+                MappedFieldType finalFieldType = fieldType;
                 builders.put(
                     fieldName,
                     hc -> new FieldHighlightContext(
-                        fieldType.name(),
+                        finalFieldType.name(),
                         field,
-                        fieldType,
+                        finalFieldType,
                         context,
                         hc,
                         highlightQuery == null ? query : highlightQuery,
diff --git a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightUtils.java b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightUtils.java
index 2238554a12149..55df80e7d7aa9 100644
--- a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightUtils.java
+++ b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/HighlightUtils.java
@@ -35,6 +35,7 @@
 import org.apache.lucene.search.highlight.Encoder;
 import org.apache.lucene.search.highlight.SimpleHTMLEncoder;
 import org.opensearch.index.fieldvisitor.CustomFieldsVisitor;
+import org.opensearch.index.mapper.DerivedFieldValueFetcher;
 import org.opensearch.index.mapper.MappedFieldType;
 import org.opensearch.index.mapper.ValueFetcher;
 import org.opensearch.index.query.QueryShardContext;
@@ -77,6 +78,9 @@ public static List<Object> loadFieldValues(
             return textsToHighlight != null ? textsToHighlight : Collections.emptyList();
         }
         ValueFetcher fetcher = fieldType.valueFetcher(context, null, null);
+        if (fetcher instanceof DerivedFieldValueFetcher) {
+            fetcher.setNextReader(hitContext.reader().getContext());
+        }
         return fetcher.fetchValues(hitContext.sourceLookup());
     }
 
diff --git a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/UnifiedHighlighter.java b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/UnifiedHighlighter.java
index df85246a84d54..c791c8bc05054 100644
--- a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/UnifiedHighlighter.java
+++ b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/UnifiedHighlighter.java
@@ -48,6 +48,7 @@
 import org.opensearch.common.CheckedSupplier;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.common.text.Text;
+import org.opensearch.index.mapper.DerivedFieldType;
 import org.opensearch.index.mapper.DocumentMapper;
 import org.opensearch.index.mapper.IdFieldMapper;
 import org.opensearch.index.mapper.MappedFieldType;
@@ -159,6 +160,10 @@ CustomUnifiedHighlighter buildHighlighter(FieldHighlightContext fieldContext) th
         Integer fieldMaxAnalyzedOffset = fieldContext.field.fieldOptions().maxAnalyzerOffset();
         int numberOfFragments = fieldContext.field.fieldOptions().numberOfFragments();
         Analyzer analyzer = getAnalyzer(fieldContext.context.mapperService().documentMapper());
+        if (fieldContext.context.getQueryShardContext().getDerivedFieldType(fieldContext.fieldName) != null) {
+            analyzer = ((DerivedFieldType) fieldContext.context.getQueryShardContext().getDerivedFieldType(fieldContext.fieldName))
+                .getIndexAnalyzer();
+        }
         if (fieldMaxAnalyzedOffset != null) {
             analyzer = getLimitedOffsetAnalyzer(analyzer, fieldMaxAnalyzedOffset);
         }
diff --git a/server/src/main/java/org/opensearch/search/internal/SearchContext.java b/server/src/main/java/org/opensearch/search/internal/SearchContext.java
index 3d13378e58e5d..0c8240d3a8322 100644
--- a/server/src/main/java/org/opensearch/search/internal/SearchContext.java
+++ b/server/src/main/java/org/opensearch/search/internal/SearchContext.java
@@ -113,6 +113,12 @@ public List<Aggregator> toAggregators(Collection<Collector> collectors) {
             // should not be called when there is no aggregation collector
             throw new IllegalStateException("Unexpected toAggregators call on NO_OP_BUCKET_COLLECTOR_PROCESSOR");
         }
+
+        @Override
+        public List<InternalAggregation> toInternalAggregations(Collection<Collector> collectors) {
+            // should not be called when there is no aggregation collector
+            throw new IllegalStateException("Unexpected toInternalAggregations call on NO_OP_BUCKET_COLLECTOR_PROCESSOR");
+        }
     };
 
     private final List<Releasable> releasables = new CopyOnWriteArrayList<>();
@@ -188,10 +194,6 @@ public final void close() {
 
     public abstract void highlight(SearchHighlightContext highlight);
 
-    public boolean hasInnerHits() {
-        return innerHitsContext != null;
-    }
-
     public InnerHitsContext innerHits() {
         if (innerHitsContext == null) {
             innerHitsContext = new InnerHitsContext();
@@ -516,4 +518,8 @@ public String toString() {
     public abstract int getTargetMaxSliceCount();
 
     public abstract boolean shouldUseTimeSeriesDescSortOptimization();
+
+    public int maxAggRewriteFilters() {
+        return 0;
+    }
 }
diff --git a/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineService.java b/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineService.java
index 2175b5d135394..012d6695c042b 100644
--- a/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineService.java
+++ b/server/src/main/java/org/opensearch/search/pipeline/SearchPipelineService.java
@@ -23,6 +23,7 @@
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.ClusterStateApplier;
 import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
 import org.opensearch.cluster.metadata.Metadata;
 import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.cluster.service.ClusterManagerTaskKeys;
@@ -35,10 +36,12 @@
 import org.opensearch.common.xcontent.XContentHelper;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
+import org.opensearch.core.index.Index;
 import org.opensearch.core.service.ReportingService;
 import org.opensearch.core.xcontent.NamedXContentRegistry;
 import org.opensearch.env.Environment;
 import org.opensearch.gateway.GatewayService;
+import org.opensearch.index.IndexNotFoundException;
 import org.opensearch.index.IndexSettings;
 import org.opensearch.index.analysis.AnalysisRegistry;
 import org.opensearch.ingest.ConfigurationUtils;
@@ -62,6 +65,8 @@
 /**
  * The main entry point for search pipelines. Handles CRUD operations and exposes the API to execute search pipelines
  * against requests and responses.
+ *
+ * @opensearch.internal
  */
 public class SearchPipelineService implements ClusterStateApplier, ReportingService<SearchPipelineInfo> {
 
@@ -360,7 +365,7 @@ static ClusterState innerDelete(DeleteSearchPipelineRequest request, ClusterStat
         return newState.build();
     }
 
-    public PipelinedRequest resolvePipeline(SearchRequest searchRequest) {
+    public PipelinedRequest resolvePipeline(SearchRequest searchRequest, IndexNameExpressionResolver indexNameExpressionResolver) {
         Pipeline pipeline = Pipeline.NO_OP_PIPELINE;
 
         if (searchRequest.source() != null && searchRequest.source().searchPipelineSource() != null) {
@@ -390,14 +395,27 @@ public PipelinedRequest resolvePipeline(SearchRequest searchRequest) {
             if (searchRequest.pipeline() != null) {
                 // Named pipeline specified for the request
                 pipelineId = searchRequest.pipeline();
-            } else if (state != null && searchRequest.indices() != null && searchRequest.indices().length == 1) {
-                // Check for index default pipeline
-                IndexMetadata indexMetadata = state.metadata().index(searchRequest.indices()[0]);
-                if (indexMetadata != null) {
-                    Settings indexSettings = indexMetadata.getSettings();
-                    if (IndexSettings.DEFAULT_SEARCH_PIPELINE.exists(indexSettings)) {
-                        pipelineId = IndexSettings.DEFAULT_SEARCH_PIPELINE.get(indexSettings);
+            } else if (state != null && searchRequest.indices() != null && searchRequest.indices().length != 0) {
+                try {
+                    // Check for index default pipeline
+                    Index[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, searchRequest);
+                    for (Index index : concreteIndices) {
+                        IndexMetadata indexMetadata = state.metadata().index(index);
+                        if (indexMetadata != null) {
+                            Settings indexSettings = indexMetadata.getSettings();
+                            if (IndexSettings.DEFAULT_SEARCH_PIPELINE.exists(indexSettings)) {
+                                String currentPipelineId = IndexSettings.DEFAULT_SEARCH_PIPELINE.get(indexSettings);
+                                if (NOOP_PIPELINE_ID.equals(pipelineId)) {
+                                    pipelineId = currentPipelineId;
+                                } else if (!pipelineId.equals(currentPipelineId)) {
+                                    pipelineId = NOOP_PIPELINE_ID;
+                                    break;
+                                }
+                            }
+                        }
                     }
+                } catch (IndexNotFoundException e) {
+                    logger.debug("Default pipeline not applied for {}", (Object) searchRequest.indices());
                 }
             }
             if (NOOP_PIPELINE_ID.equals(pipelineId) == false) {
diff --git a/server/src/main/java/org/opensearch/search/query/ConcurrentQueryPhaseSearcher.java b/server/src/main/java/org/opensearch/search/query/ConcurrentQueryPhaseSearcher.java
index e22f766d3894c..771ac60dfb5e5 100644
--- a/server/src/main/java/org/opensearch/search/query/ConcurrentQueryPhaseSearcher.java
+++ b/server/src/main/java/org/opensearch/search/query/ConcurrentQueryPhaseSearcher.java
@@ -23,10 +23,9 @@
 
 import java.io.IOException;
 import java.util.LinkedList;
+import java.util.Objects;
 import java.util.concurrent.ExecutionException;
 
-import static org.opensearch.search.query.TopDocsCollectorContext.createTopDocsCollectorContext;
-
 /**
  * The implementation of the {@link QueryPhaseSearcher} which attempts to use concurrent
  * search of Apache Lucene segments if it has been enabled.
@@ -46,10 +45,19 @@ protected boolean searchWithCollector(
         ContextIndexSearcher searcher,
         Query query,
         LinkedList<QueryCollectorContext> collectors,
+        QueryCollectorContext queryCollectorContext,
         boolean hasFilterCollector,
         boolean hasTimeout
     ) throws IOException {
-        return searchWithCollectorManager(searchContext, searcher, query, collectors, hasFilterCollector, hasTimeout);
+        return searchWithCollectorManager(
+            searchContext,
+            searcher,
+            query,
+            collectors,
+            queryCollectorContext,
+            hasFilterCollector,
+            hasTimeout
+        );
     }
 
     private static boolean searchWithCollectorManager(
@@ -57,13 +65,12 @@ private static boolean searchWithCollectorManager(
         ContextIndexSearcher searcher,
         Query query,
         LinkedList<QueryCollectorContext> collectorContexts,
+        QueryCollectorContext queryCollectorContext,
         boolean hasFilterCollector,
         boolean timeoutSet
     ) throws IOException {
-        // create the top docs collector last when the other collectors are known
-        final TopDocsCollectorContext topDocsFactory = createTopDocsCollectorContext(searchContext, hasFilterCollector);
-        // add the top docs collector, the first collector context in the chain
-        collectorContexts.addFirst(topDocsFactory);
+        // add the passed collector, the first collector context in the chain
+        collectorContexts.addFirst(Objects.requireNonNull(queryCollectorContext));
 
         final QuerySearchResult queryResult = searchContext.queryResult();
         final CollectorManager<?, ReduceableSearchResult> collectorManager;
@@ -95,7 +102,10 @@ private static boolean searchWithCollectorManager(
             queryResult.terminatedEarly(false);
         }
 
-        return topDocsFactory.shouldRescore();
+        if (queryCollectorContext instanceof RescoringQueryCollectorContext) {
+            return ((RescoringQueryCollectorContext) queryCollectorContext).shouldRescore();
+        }
+        return false;
     }
 
     @Override
diff --git a/server/src/main/java/org/opensearch/search/query/QueryCollectorContext.java b/server/src/main/java/org/opensearch/search/query/QueryCollectorContext.java
index 91762bee2ac08..08b048cf682bb 100644
--- a/server/src/main/java/org/opensearch/search/query/QueryCollectorContext.java
+++ b/server/src/main/java/org/opensearch/search/query/QueryCollectorContext.java
@@ -77,6 +77,29 @@ public ScoreMode scoreMode() {
         }
     };
 
+    public static final QueryCollectorContext EMPTY_CONTEXT = new QueryCollectorContext("empty") {
+
+        @Override
+        Collector create(Collector in) throws IOException {
+            return EMPTY_COLLECTOR;
+        }
+
+        @Override
+        CollectorManager<?, ReduceableSearchResult> createManager(CollectorManager<?, ReduceableSearchResult> in) throws IOException {
+            return new CollectorManager<Collector, ReduceableSearchResult>() {
+                @Override
+                public Collector newCollector() throws IOException {
+                    return EMPTY_COLLECTOR;
+                }
+
+                @Override
+                public ReduceableSearchResult reduce(Collection<Collector> collectors) throws IOException {
+                    return result -> {};
+                }
+            };
+        }
+    };
+
     private String profilerName;
 
     QueryCollectorContext(String profilerName) {
diff --git a/server/src/main/java/org/opensearch/search/query/QueryPhase.java b/server/src/main/java/org/opensearch/search/query/QueryPhase.java
index 8f98f0d9efbd4..608649ad22b23 100644
--- a/server/src/main/java/org/opensearch/search/query/QueryPhase.java
+++ b/server/src/main/java/org/opensearch/search/query/QueryPhase.java
@@ -335,13 +335,12 @@ private static boolean searchWithCollector(
         ContextIndexSearcher searcher,
         Query query,
         LinkedList<QueryCollectorContext> collectors,
+        QueryCollectorContext queryCollectorContext,
         boolean hasFilterCollector,
         boolean timeoutSet
     ) throws IOException {
-        // create the top docs collector last when the other collectors are known
-        final TopDocsCollectorContext topDocsFactory = createTopDocsCollectorContext(searchContext, hasFilterCollector);
-        // add the top docs collector, the first collector context in the chain
-        collectors.addFirst(topDocsFactory);
+        // add passed collector, the first collector context in the chain
+        collectors.addFirst(Objects.requireNonNull(queryCollectorContext));
 
         final Collector queryCollector;
         if (searchContext.getProfilers() != null) {
@@ -370,7 +369,10 @@ private static boolean searchWithCollector(
         for (QueryCollectorContext ctx : collectors) {
             ctx.postProcess(queryResult);
         }
-        return topDocsFactory.shouldRescore();
+        if (queryCollectorContext instanceof RescoringQueryCollectorContext) {
+            return ((RescoringQueryCollectorContext) queryCollectorContext).shouldRescore();
+        }
+        return false;
     }
 
     /**
@@ -440,7 +442,29 @@ protected boolean searchWithCollector(
             boolean hasFilterCollector,
             boolean hasTimeout
         ) throws IOException {
-            return QueryPhase.searchWithCollector(searchContext, searcher, query, collectors, hasFilterCollector, hasTimeout);
+            // create the top docs collector last when the other collectors are known
+            final TopDocsCollectorContext topDocsFactory = createTopDocsCollectorContext(searchContext, hasFilterCollector);
+            return searchWithCollector(searchContext, searcher, query, collectors, topDocsFactory, hasFilterCollector, hasTimeout);
+        }
+
+        protected boolean searchWithCollector(
+            SearchContext searchContext,
+            ContextIndexSearcher searcher,
+            Query query,
+            LinkedList<QueryCollectorContext> collectors,
+            QueryCollectorContext queryCollectorContext,
+            boolean hasFilterCollector,
+            boolean hasTimeout
+        ) throws IOException {
+            return QueryPhase.searchWithCollector(
+                searchContext,
+                searcher,
+                query,
+                collectors,
+                queryCollectorContext,
+                hasFilterCollector,
+                hasTimeout
+            );
         }
     }
 }
diff --git a/server/src/main/java/org/opensearch/search/query/RescoringQueryCollectorContext.java b/server/src/main/java/org/opensearch/search/query/RescoringQueryCollectorContext.java
new file mode 100644
index 0000000000000..dc47ee80dba42
--- /dev/null
+++ b/server/src/main/java/org/opensearch/search/query/RescoringQueryCollectorContext.java
@@ -0,0 +1,27 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.search.query;
+
+import org.opensearch.common.annotation.PublicApi;
+
+/**
+ * Abstraction that allows indication of whether results should be rescored or not based on
+ * custom logic of exact {@link QueryCollectorContext} implementation.
+ *
+ * @opensearch.api
+ */
+@PublicApi(since = "2.15.0")
+public interface RescoringQueryCollectorContext {
+
+    /**
+     * Indicates if results from the query context should be rescored
+     * @return true if results must be rescored, false otherwise
+     */
+    boolean shouldRescore();
+}
diff --git a/server/src/main/java/org/opensearch/search/query/TopDocsCollectorContext.java b/server/src/main/java/org/opensearch/search/query/TopDocsCollectorContext.java
index 65d3948c8401e..f780f6fe32af2 100644
--- a/server/src/main/java/org/opensearch/search/query/TopDocsCollectorContext.java
+++ b/server/src/main/java/org/opensearch/search/query/TopDocsCollectorContext.java
@@ -95,7 +95,7 @@
  *
  * @opensearch.internal
  */
-public abstract class TopDocsCollectorContext extends QueryCollectorContext {
+public abstract class TopDocsCollectorContext extends QueryCollectorContext implements RescoringQueryCollectorContext {
     protected final int numHits;
 
     TopDocsCollectorContext(String profilerName, int numHits) {
diff --git a/server/src/main/java/org/opensearch/snapshots/RestoreService.java b/server/src/main/java/org/opensearch/snapshots/RestoreService.java
index e5ac604e0a5e3..5883a8a37be71 100644
--- a/server/src/main/java/org/opensearch/snapshots/RestoreService.java
+++ b/server/src/main/java/org/opensearch/snapshots/RestoreService.java
@@ -79,6 +79,7 @@
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
+import org.opensearch.common.util.ArrayUtils;
 import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.index.Index;
@@ -90,6 +91,8 @@
 import org.opensearch.index.store.remote.filecache.FileCacheStats;
 import org.opensearch.indices.IndicesService;
 import org.opensearch.indices.ShardLimitValidator;
+import org.opensearch.node.remotestore.RemoteStoreNodeAttribute;
+import org.opensearch.node.remotestore.RemoteStoreNodeService;
 import org.opensearch.repositories.IndexId;
 import org.opensearch.repositories.RepositoriesService;
 import org.opensearch.repositories.Repository;
@@ -116,16 +119,18 @@
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_INDEX_UUID;
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS;
+import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY;
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED;
+import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY;
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_VERSION_CREATED;
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_VERSION_UPGRADED;
 import static org.opensearch.common.util.FeatureFlags.SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY;
+import static org.opensearch.common.util.IndexUtils.filterIndices;
 import static org.opensearch.common.util.set.Sets.newHashSet;
 import static org.opensearch.index.IndexModule.INDEX_STORE_TYPE_SETTING;
 import static org.opensearch.index.store.remote.directory.RemoteSnapshotDirectory.SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY_MINIMUM_VERSION;
 import static org.opensearch.index.store.remote.filecache.FileCache.DATA_TO_FILE_CACHE_SIZE_RATIO_SETTING;
 import static org.opensearch.node.Node.NODE_SEARCH_CACHE_SIZE_SETTING;
-import static org.opensearch.snapshots.SnapshotUtils.filterIndices;
 
 /**
  * Service responsible for restoring snapshots
@@ -153,20 +158,30 @@ public class RestoreService implements ClusterStateApplier {
 
     private static final Logger logger = LogManager.getLogger(RestoreService.class);
 
-    private static final Set<String> UNMODIFIABLE_SETTINGS = unmodifiableSet(
-        newHashSet(SETTING_NUMBER_OF_SHARDS, SETTING_VERSION_CREATED, SETTING_INDEX_UUID, SETTING_CREATION_DATE, SETTING_HISTORY_UUID)
+    private static final Set<String> USER_UNMODIFIABLE_SETTINGS = unmodifiableSet(
+        newHashSet(
+            SETTING_NUMBER_OF_SHARDS,
+            SETTING_VERSION_CREATED,
+            SETTING_INDEX_UUID,
+            SETTING_CREATION_DATE,
+            SETTING_HISTORY_UUID,
+            SETTING_REMOTE_STORE_ENABLED,
+            SETTING_REMOTE_SEGMENT_STORE_REPOSITORY,
+            SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY
+        )
     );
 
     // It's OK to change some settings, but we shouldn't allow simply removing them
-    private static final Set<String> UNREMOVABLE_SETTINGS;
+    private static final Set<String> USER_UNREMOVABLE_SETTINGS;
+    private static final String REMOTE_STORE_INDEX_SETTINGS_REGEX = "index.remote_store.*";
 
     static {
-        Set<String> unremovable = new HashSet<>(UNMODIFIABLE_SETTINGS.size() + 4);
-        unremovable.addAll(UNMODIFIABLE_SETTINGS);
+        Set<String> unremovable = new HashSet<>(USER_UNMODIFIABLE_SETTINGS.size() + 4);
+        unremovable.addAll(USER_UNMODIFIABLE_SETTINGS);
         unremovable.add(SETTING_NUMBER_OF_REPLICAS);
         unremovable.add(SETTING_AUTO_EXPAND_REPLICAS);
         unremovable.add(SETTING_VERSION_UPGRADED);
-        UNREMOVABLE_SETTINGS = unmodifiableSet(unremovable);
+        USER_UNREMOVABLE_SETTINGS = unmodifiableSet(unremovable);
     }
 
     private final ClusterService clusterService;
@@ -370,10 +385,16 @@ public ClusterState execute(ClusterState currentState) {
                                 boolean partial = checkPartial(index);
 
                                 IndexId snapshotIndexId = repositoryData.resolveIndexId(index);
+
+                                final Settings overrideSettingsInternal = getOverrideSettingsInternal();
+                                final String[] ignoreSettingsInternal = getIgnoreSettingsInternal();
+
                                 IndexMetadata snapshotIndexMetadata = updateIndexSettings(
                                     metadata.index(index),
                                     request.indexSettings(),
-                                    request.ignoreIndexSettings()
+                                    request.ignoreIndexSettings(),
+                                    overrideSettingsInternal,
+                                    ignoreSettingsInternal
                                 );
                                 if (isRemoteSnapshot) {
                                     snapshotIndexMetadata = addSnapshotToIndexSettings(snapshotIndexMetadata, snapshot, snapshotIndexId);
@@ -434,7 +455,7 @@ public ClusterState execute(ClusterState currentState) {
                                 final Index renamedIndex;
                                 if (currentIndexMetadata == null) {
                                     // Index doesn't exist - create it and start recovery
-                                    // Make sure that the index we are about to create has a validate name
+                                    // Make sure that the index we are about to create has valid name
                                     boolean isHidden = IndexMetadata.INDEX_HIDDEN_SETTING.get(snapshotIndexMetadata.getSettings());
                                     createIndexService.validateIndexName(renamedIndexName, currentState);
                                     createIndexService.validateDotIndex(renamedIndexName, isHidden);
@@ -451,7 +472,7 @@ public ClusterState execute(ClusterState currentState) {
                                             .put(snapshotIndexMetadata.getSettings())
                                             .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
                                     );
-                                    createIndexService.addRemoteCustomData(indexMdBuilder);
+                                    createIndexService.addRemoteStorePathStrategyInCustomData(indexMdBuilder, false);
                                     shardLimitValidator.validateShardLimit(
                                         renamedIndexName,
                                         snapshotIndexMetadata.getSettings(),
@@ -640,6 +661,48 @@ private void checkAliasNameConflicts(Map<String, String> renamedIndices, Set<Str
                         }
                     }
 
+                    private String[] getIgnoreSettingsInternal() {
+                        // for non-remote store enabled domain, we will remove all the remote store
+                        // related index settings present in the snapshot.
+                        String[] indexSettingsToBeIgnored = new String[] {};
+                        if (false == RemoteStoreNodeAttribute.isRemoteStoreAttributePresent(clusterService.getSettings())) {
+                            indexSettingsToBeIgnored = ArrayUtils.concat(
+                                indexSettingsToBeIgnored,
+                                new String[] { REMOTE_STORE_INDEX_SETTINGS_REGEX }
+                            );
+                        }
+                        return indexSettingsToBeIgnored;
+                    }
+
+                    private Settings getOverrideSettingsInternal() {
+                        final Settings.Builder settingsBuilder = Settings.builder();
+
+                        // We will use whatever replication strategy provided by user or from snapshot metadata unless
+                        // cluster is remote store enabled or user have restricted a specific replication type in the
+                        // cluster. If cluster is undergoing remote store migration, replication strategy is strictly SEGMENT type
+                        if (RemoteStoreNodeAttribute.isRemoteStoreAttributePresent(clusterService.getSettings())
+                            || clusterSettings.get(IndicesService.CLUSTER_INDEX_RESTRICT_REPLICATION_TYPE_SETTING)
+                            || RemoteStoreNodeService.isMigratingToRemoteStore(clusterSettings)) {
+                            MetadataCreateIndexService.updateReplicationStrategy(
+                                settingsBuilder,
+                                request.indexSettings(),
+                                clusterService.getSettings(),
+                                null,
+                                clusterSettings
+                            );
+                        }
+                        // remote store settings needs to be overridden if the remote store feature is enabled in the
+                        // cluster where snapshot is being restored.
+                        MetadataCreateIndexService.updateRemoteStoreSettings(
+                            settingsBuilder,
+                            clusterService.state(),
+                            clusterSettings,
+                            clusterService.getSettings(),
+                            request.getDescription()
+                        );
+                        return settingsBuilder.build();
+                    }
+
                     private void populateIgnoredShards(String index, final Set<Integer> ignoreShards) {
                         for (SnapshotShardFailure failure : snapshotInfo.shardFailures()) {
                             if (index.equals(failure.index())) {
@@ -712,16 +775,18 @@ private void validateExistingIndex(
                      */
                     private IndexMetadata updateIndexSettings(
                         IndexMetadata indexMetadata,
-                        Settings changeSettings,
-                        String[] ignoreSettings
+                        Settings overrideSettings,
+                        String[] ignoreSettings,
+                        Settings overrideSettingsInternal,
+                        String[] ignoreSettingsInternal
                     ) {
                         Settings normalizedChangeSettings = Settings.builder()
-                            .put(changeSettings)
+                            .put(overrideSettings)
                             .normalizePrefix(IndexMetadata.INDEX_SETTING_PREFIX)
                             .build();
                         if (IndexSettings.INDEX_SOFT_DELETES_SETTING.get(indexMetadata.getSettings())
-                            && IndexSettings.INDEX_SOFT_DELETES_SETTING.exists(changeSettings)
-                            && IndexSettings.INDEX_SOFT_DELETES_SETTING.get(changeSettings) == false) {
+                            && IndexSettings.INDEX_SOFT_DELETES_SETTING.exists(overrideSettings)
+                            && IndexSettings.INDEX_SOFT_DELETES_SETTING.get(overrideSettings) == false) {
                             throw new SnapshotRestoreException(
                                 snapshot,
                                 "cannot disable setting [" + IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey() + "] on restore"
@@ -733,7 +798,7 @@ private IndexMetadata updateIndexSettings(
                         List<String> simpleMatchPatterns = new ArrayList<>();
                         for (String ignoredSetting : ignoreSettings) {
                             if (!Regex.isSimpleMatchPattern(ignoredSetting)) {
-                                if (UNREMOVABLE_SETTINGS.contains(ignoredSetting)) {
+                                if (USER_UNREMOVABLE_SETTINGS.contains(ignoredSetting)) {
                                     throw new SnapshotRestoreException(
                                         snapshot,
                                         "cannot remove setting [" + ignoredSetting + "] on restore"
@@ -745,8 +810,18 @@ private IndexMetadata updateIndexSettings(
                                 simpleMatchPatterns.add(ignoredSetting);
                             }
                         }
+
+                        // add internal settings to ignore settings list
+                        for (String ignoredSetting : ignoreSettingsInternal) {
+                            if (!Regex.isSimpleMatchPattern(ignoredSetting)) {
+                                keyFilters.add(ignoredSetting);
+                            } else {
+                                simpleMatchPatterns.add(ignoredSetting);
+                            }
+                        }
+
                         Predicate<String> settingsFilter = k -> {
-                            if (UNREMOVABLE_SETTINGS.contains(k) == false) {
+                            if (USER_UNREMOVABLE_SETTINGS.contains(k) == false) {
                                 for (String filterKey : keyFilters) {
                                     if (k.equals(filterKey)) {
                                         return false;
@@ -763,12 +838,17 @@ private IndexMetadata updateIndexSettings(
                         Settings.Builder settingsBuilder = Settings.builder()
                             .put(settings.filter(settingsFilter))
                             .put(normalizedChangeSettings.filter(k -> {
-                                if (UNMODIFIABLE_SETTINGS.contains(k)) {
+                                if (USER_UNMODIFIABLE_SETTINGS.contains(k)) {
                                     throw new SnapshotRestoreException(snapshot, "cannot modify setting [" + k + "] on restore");
                                 } else {
                                     return true;
                                 }
                             }));
+
+                        // override internal settings
+                        if (overrideSettingsInternal != null) {
+                            settingsBuilder.put(overrideSettingsInternal).normalizePrefix(IndexMetadata.INDEX_SETTING_PREFIX);
+                        }
                         settingsBuilder.remove(MetadataIndexStateService.VERIFIED_BEFORE_CLOSE_SETTING.getKey());
                         return builder.settings(settingsBuilder).build();
                     }
diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotUtils.java b/server/src/main/java/org/opensearch/snapshots/SnapshotUtils.java
index e7338a29cafeb..8cce5e1c98254 100644
--- a/server/src/main/java/org/opensearch/snapshots/SnapshotUtils.java
+++ b/server/src/main/java/org/opensearch/snapshots/SnapshotUtils.java
@@ -31,24 +31,15 @@
 
 package org.opensearch.snapshots;
 
-import org.opensearch.action.support.IndicesOptions;
 import org.opensearch.cluster.metadata.IndexMetadata;
-import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
-import org.opensearch.common.regex.Regex;
 import org.opensearch.index.IndexModule;
-import org.opensearch.index.IndexNotFoundException;
 import org.opensearch.index.IndexSettings;
 
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 /**
  * Snapshot utilities
@@ -57,102 +48,6 @@
  */
 public class SnapshotUtils {
 
-    /**
-     * Filters out list of available indices based on the list of selected indices.
-     *
-     * @param availableIndices list of available indices
-     * @param selectedIndices  list of selected indices
-     * @param indicesOptions    ignore indices flag
-     * @return filtered out indices
-     */
-    public static List<String> filterIndices(List<String> availableIndices, String[] selectedIndices, IndicesOptions indicesOptions) {
-        if (IndexNameExpressionResolver.isAllIndices(Arrays.asList(selectedIndices))) {
-            return availableIndices;
-        }
-
-        // Move the exclusions to end of list to ensure they are processed
-        // after explicitly selected indices are chosen.
-        final List<String> excludesAtEndSelectedIndices = Stream.concat(
-            Arrays.stream(selectedIndices).filter(s -> s.isEmpty() || s.charAt(0) != '-'),
-            Arrays.stream(selectedIndices).filter(s -> !s.isEmpty() && s.charAt(0) == '-')
-        ).collect(Collectors.toUnmodifiableList());
-
-        Set<String> result = null;
-        for (int i = 0; i < excludesAtEndSelectedIndices.size(); i++) {
-            String indexOrPattern = excludesAtEndSelectedIndices.get(i);
-            boolean add = true;
-            if (!indexOrPattern.isEmpty()) {
-                if (availableIndices.contains(indexOrPattern)) {
-                    if (result == null) {
-                        result = new HashSet<>();
-                    }
-                    result.add(indexOrPattern);
-                    continue;
-                }
-                if (indexOrPattern.charAt(0) == '+') {
-                    add = true;
-                    indexOrPattern = indexOrPattern.substring(1);
-                    // if its the first, add empty set
-                    if (i == 0) {
-                        result = new HashSet<>();
-                    }
-                } else if (indexOrPattern.charAt(0) == '-') {
-                    // If the first index pattern is an exclusion, then all patterns are exclusions due to the
-                    // reordering logic above. In this case, the request is interpreted as "include all indexes except
-                    // those matching the exclusions" so we add all indices here and then remove the ones that match the exclusion patterns.
-                    if (i == 0) {
-                        result = new HashSet<>(availableIndices);
-                    }
-                    add = false;
-                    indexOrPattern = indexOrPattern.substring(1);
-                }
-            }
-            if (indexOrPattern.isEmpty() || !Regex.isSimpleMatchPattern(indexOrPattern)) {
-                if (!availableIndices.contains(indexOrPattern)) {
-                    if (!indicesOptions.ignoreUnavailable()) {
-                        throw new IndexNotFoundException(indexOrPattern);
-                    } else {
-                        if (result == null) {
-                            // add all the previous ones...
-                            result = new HashSet<>(availableIndices.subList(0, i));
-                        }
-                    }
-                } else {
-                    if (result != null) {
-                        if (add) {
-                            result.add(indexOrPattern);
-                        } else {
-                            result.remove(indexOrPattern);
-                        }
-                    }
-                }
-                continue;
-            }
-            if (result == null) {
-                // add all the previous ones...
-                result = new HashSet<>(availableIndices.subList(0, i));
-            }
-            boolean found = false;
-            for (String index : availableIndices) {
-                if (Regex.simpleMatch(indexOrPattern, index)) {
-                    found = true;
-                    if (add) {
-                        result.add(index);
-                    } else {
-                        result.remove(index);
-                    }
-                }
-            }
-            if (!found && !indicesOptions.allowNoIndices()) {
-                throw new IndexNotFoundException(indexOrPattern);
-            }
-        }
-        if (result == null) {
-            return Collections.unmodifiableList(new ArrayList<>(Arrays.asList(selectedIndices)));
-        }
-        return Collections.unmodifiableList(new ArrayList<>(result));
-    }
-
     /**
      * Validates if there are any remote snapshots backing an index
      * @param metadata index metadata from cluster state
diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java
index 71918bc73b55a..acc2dc83749cd 100644
--- a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java
+++ b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java
@@ -131,6 +131,9 @@
 import static java.util.Collections.emptySet;
 import static java.util.Collections.unmodifiableList;
 import static org.opensearch.cluster.SnapshotsInProgress.completed;
+import static org.opensearch.common.util.IndexUtils.filterIndices;
+import static org.opensearch.node.remotestore.RemoteStoreNodeService.CompatibilityMode;
+import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING;
 import static org.opensearch.repositories.blobstore.BlobStoreRepository.REMOTE_STORE_INDEX_SHALLOW_COPY;
 import static org.opensearch.snapshots.SnapshotUtils.validateSnapshotsBackingAnyIndex;
 
@@ -343,6 +346,13 @@ public ClusterState execute(ClusterState currentState) {
                 }
 
                 boolean remoteStoreIndexShallowCopy = REMOTE_STORE_INDEX_SHALLOW_COPY.get(repository.getMetadata().settings());
+                logger.debug("remote_store_index_shallow_copy setting is set as [{}]", remoteStoreIndexShallowCopy);
+                if (remoteStoreIndexShallowCopy
+                    && clusterService.getClusterSettings().get(REMOTE_STORE_COMPATIBILITY_MODE_SETTING).equals(CompatibilityMode.MIXED)) {
+                    // don't allow shallow snapshots if compatibility mode is not strict
+                    logger.warn("Shallow snapshots are not supported during migration. Falling back to full snapshot.");
+                    remoteStoreIndexShallowCopy = false;
+                }
                 newEntry = SnapshotsInProgress.startedEntry(
                     new Snapshot(repositoryName, snapshotId),
                     request.includeGlobalState(),
@@ -466,11 +476,7 @@ public ClusterState execute(ClusterState currentState) {
                         indicesForSnapshot.add(indexId.getName());
                     }
                 }
-                final List<String> matchingIndices = SnapshotUtils.filterIndices(
-                    indicesForSnapshot,
-                    request.indices(),
-                    request.indicesOptions()
-                );
+                final List<String> matchingIndices = filterIndices(indicesForSnapshot, request.indices(), request.indicesOptions());
                 if (matchingIndices.isEmpty()) {
                     throw new SnapshotException(
                         new Snapshot(repositoryName, sourceSnapshotId),
diff --git a/server/src/main/java/org/opensearch/tasks/TaskListener.java b/server/src/main/java/org/opensearch/tasks/TaskListener.java
index 97df8eacee584..7ec08984a7f07 100644
--- a/server/src/main/java/org/opensearch/tasks/TaskListener.java
+++ b/server/src/main/java/org/opensearch/tasks/TaskListener.java
@@ -32,11 +32,14 @@
 
 package org.opensearch.tasks;
 
+import org.opensearch.common.annotation.PublicApi;
+
 /**
  * Listener for Task success or failure.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface TaskListener<Response> {
     /**
      * Handle task response. This response may constitute a failure or a success
diff --git a/server/src/main/java/org/opensearch/transport/ConnectionProfile.java b/server/src/main/java/org/opensearch/transport/ConnectionProfile.java
index b9764c0c53f4a..931707e4a1cdc 100644
--- a/server/src/main/java/org/opensearch/transport/ConnectionProfile.java
+++ b/server/src/main/java/org/opensearch/transport/ConnectionProfile.java
@@ -33,6 +33,7 @@
 
 import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.common.Nullable;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
 
@@ -49,8 +50,9 @@
  * A connection profile describes how many connection are established to specific node for each of the available request types.
  * ({@link org.opensearch.transport.TransportRequestOptions.Type}). This allows to tailor a connection towards a specific usage.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class ConnectionProfile {
 
     /**
diff --git a/server/src/main/java/org/opensearch/transport/Header.java b/server/src/main/java/org/opensearch/transport/Header.java
index a179cfb35288e..ac30df8dda02c 100644
--- a/server/src/main/java/org/opensearch/transport/Header.java
+++ b/server/src/main/java/org/opensearch/transport/Header.java
@@ -33,6 +33,7 @@
 package org.opensearch.transport;
 
 import org.opensearch.Version;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.collect.Tuple;
 import org.opensearch.common.util.concurrent.ThreadContext;
 import org.opensearch.core.common.io.stream.StreamInput;
@@ -47,8 +48,9 @@
 /**
  * Transport Header
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class Header {
 
     private static final String RESPONSE_NAME = "NO_ACTION_NAME_FOR_RESPONSES";
@@ -73,11 +75,11 @@ public int getNetworkMessageSize() {
         return networkMessageSize;
     }
 
-    Version getVersion() {
+    public Version getVersion() {
         return version;
     }
 
-    long getRequestId() {
+    public long getRequestId() {
         return requestId;
     }
 
@@ -85,7 +87,7 @@ byte getStatus() {
         return status;
     }
 
-    boolean isRequest() {
+    public boolean isRequest() {
         return TransportStatus.isRequest(status);
     }
 
@@ -97,7 +99,7 @@ boolean isError() {
         return TransportStatus.isError(status);
     }
 
-    boolean isHandshake() {
+    public boolean isHandshake() {
         return TransportStatus.isHandshake(status);
     }
 
diff --git a/server/src/main/java/org/opensearch/transport/InboundAggregator.java b/server/src/main/java/org/opensearch/transport/InboundAggregator.java
index e894331f3b64e..f52875d880b4f 100644
--- a/server/src/main/java/org/opensearch/transport/InboundAggregator.java
+++ b/server/src/main/java/org/opensearch/transport/InboundAggregator.java
@@ -40,6 +40,7 @@
 import org.opensearch.core.common.bytes.BytesArray;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.core.common.bytes.CompositeBytesReference;
+import org.opensearch.transport.nativeprotocol.NativeInboundMessage;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -113,7 +114,7 @@ public void aggregate(ReleasableBytesReference content) {
         }
     }
 
-    public InboundMessage finishAggregation() throws IOException {
+    public NativeInboundMessage finishAggregation() throws IOException {
         ensureOpen();
         final ReleasableBytesReference releasableContent;
         if (isFirstContent()) {
@@ -127,7 +128,7 @@ public InboundMessage finishAggregation() throws IOException {
         }
 
         final BreakerControl breakerControl = new BreakerControl(circuitBreaker);
-        final InboundMessage aggregated = new InboundMessage(currentHeader, releasableContent, breakerControl);
+        final NativeInboundMessage aggregated = new NativeInboundMessage(currentHeader, releasableContent, breakerControl);
         boolean success = false;
         try {
             if (aggregated.getHeader().needsToReadVariableHeader()) {
@@ -142,7 +143,7 @@ public InboundMessage finishAggregation() throws IOException {
             if (isShortCircuited()) {
                 aggregated.close();
                 success = true;
-                return new InboundMessage(aggregated.getHeader(), aggregationException);
+                return new NativeInboundMessage(aggregated.getHeader(), aggregationException);
             } else {
                 success = true;
                 return aggregated;
diff --git a/server/src/main/java/org/opensearch/transport/InboundBytesHandler.java b/server/src/main/java/org/opensearch/transport/InboundBytesHandler.java
new file mode 100644
index 0000000000000..276891212e43f
--- /dev/null
+++ b/server/src/main/java/org/opensearch/transport/InboundBytesHandler.java
@@ -0,0 +1,32 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.transport;
+
+import org.opensearch.common.bytes.ReleasableBytesReference;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.function.BiConsumer;
+
+/**
+ * Interface for handling inbound bytes. Can be implemented by different transport protocols.
+ */
+public interface InboundBytesHandler extends Closeable {
+
+    public void doHandleBytes(
+        TcpChannel channel,
+        ReleasableBytesReference reference,
+        BiConsumer<TcpChannel, ProtocolInboundMessage> messageHandler
+    ) throws IOException;
+
+    public boolean canHandleBytes(ReleasableBytesReference reference);
+
+    @Override
+    void close();
+}
diff --git a/server/src/main/java/org/opensearch/transport/InboundDecoder.java b/server/src/main/java/org/opensearch/transport/InboundDecoder.java
index 82fc09a985446..d6b7a98e876b3 100644
--- a/server/src/main/java/org/opensearch/transport/InboundDecoder.java
+++ b/server/src/main/java/org/opensearch/transport/InboundDecoder.java
@@ -50,8 +50,8 @@
  */
 public class InboundDecoder implements Releasable {
 
-    static final Object PING = new Object();
-    static final Object END_CONTENT = new Object();
+    public static final Object PING = new Object();
+    public static final Object END_CONTENT = new Object();
 
     private final Version version;
     private final PageCacheRecycler recycler;
diff --git a/server/src/main/java/org/opensearch/transport/InboundHandler.java b/server/src/main/java/org/opensearch/transport/InboundHandler.java
index a8315c3cae4e0..f77c44ea362cf 100644
--- a/server/src/main/java/org/opensearch/transport/InboundHandler.java
+++ b/server/src/main/java/org/opensearch/transport/InboundHandler.java
@@ -32,35 +32,16 @@
 
 package org.opensearch.transport;
 
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.apache.lucene.util.BytesRef;
 import org.opensearch.Version;
 import org.opensearch.common.unit.TimeValue;
-import org.opensearch.common.util.concurrent.AbstractRunnable;
-import org.opensearch.common.util.concurrent.ThreadContext;
-import org.opensearch.core.common.io.stream.ByteBufferStreamInput;
-import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput;
+import org.opensearch.common.util.BigArrays;
 import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
-import org.opensearch.core.common.io.stream.StreamInput;
-import org.opensearch.core.common.transport.TransportAddress;
-import org.opensearch.core.transport.TransportResponse;
-import org.opensearch.telemetry.tracing.Span;
-import org.opensearch.telemetry.tracing.SpanBuilder;
-import org.opensearch.telemetry.tracing.SpanScope;
 import org.opensearch.telemetry.tracing.Tracer;
-import org.opensearch.telemetry.tracing.channels.TraceableTcpTransportChannel;
 import org.opensearch.threadpool.ThreadPool;
+import org.opensearch.transport.nativeprotocol.NativeInboundMessage;
 
-import java.io.EOFException;
 import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.nio.ByteBuffer;
-import java.util.Collection;
-import java.util.Collections;
 import java.util.Map;
-import java.util.stream.Collectors;
 
 /**
  * Handler for inbound data
@@ -69,24 +50,21 @@
  */
 public class InboundHandler {
 
-    private static final Logger logger = LogManager.getLogger(InboundHandler.class);
-
     private final ThreadPool threadPool;
-    private final OutboundHandler outboundHandler;
-    private final NamedWriteableRegistry namedWriteableRegistry;
-    private final TransportHandshaker handshaker;
-    private final TransportKeepAlive keepAlive;
-    private final Transport.ResponseHandlers responseHandlers;
-    private final Transport.RequestHandlers requestHandlers;
 
     private volatile TransportMessageListener messageListener = TransportMessageListener.NOOP_LISTENER;
 
     private volatile long slowLogThresholdMs = Long.MAX_VALUE;
 
-    private final Tracer tracer;
+    private final Map<String, ProtocolMessageHandler> protocolMessageHandlers;
 
     InboundHandler(
+        String nodeName,
+        Version version,
+        String[] features,
+        StatsTracker statsTracker,
         ThreadPool threadPool,
+        BigArrays bigArrays,
         OutboundHandler outboundHandler,
         NamedWriteableRegistry namedWriteableRegistry,
         TransportHandshaker handshaker,
@@ -96,16 +74,28 @@ public class InboundHandler {
         Tracer tracer
     ) {
         this.threadPool = threadPool;
-        this.outboundHandler = outboundHandler;
-        this.namedWriteableRegistry = namedWriteableRegistry;
-        this.handshaker = handshaker;
-        this.keepAlive = keepAlive;
-        this.requestHandlers = requestHandlers;
-        this.responseHandlers = responseHandlers;
-        this.tracer = tracer;
+        this.protocolMessageHandlers = Map.of(
+            NativeInboundMessage.NATIVE_PROTOCOL,
+            new NativeMessageHandler(
+                nodeName,
+                version,
+                features,
+                statsTracker,
+                threadPool,
+                bigArrays,
+                outboundHandler,
+                namedWriteableRegistry,
+                handshaker,
+                requestHandlers,
+                responseHandlers,
+                tracer,
+                keepAlive
+            )
+        );
     }
 
     void setMessageListener(TransportMessageListener listener) {
+        protocolMessageHandlers.values().forEach(handler -> handler.setMessageListener(listener));
         if (messageListener == TransportMessageListener.NOOP_LISTENER) {
             messageListener = listener;
         } else {
@@ -117,377 +107,17 @@ void setSlowLogThreshold(TimeValue slowLogThreshold) {
         this.slowLogThresholdMs = slowLogThreshold.getMillis();
     }
 
-    void inboundMessage(TcpChannel channel, InboundMessage message) throws Exception {
+    void inboundMessage(TcpChannel channel, ProtocolInboundMessage message) throws Exception {
         final long startTime = threadPool.relativeTimeInMillis();
         channel.getChannelStats().markAccessed(startTime);
-        TransportLogger.logInboundMessage(channel, message);
-        if (message.isPing()) {
-            keepAlive.receiveKeepAlive(channel);
-        } else {
-            messageReceived(channel, message, startTime);
-        }
-    }
-
-    // Empty stream constant to avoid instantiating a new stream for empty messages.
-    private static final StreamInput EMPTY_STREAM_INPUT = new ByteBufferStreamInput(ByteBuffer.wrap(BytesRef.EMPTY_BYTES));
-
-    private void messageReceived(TcpChannel channel, InboundMessage message, long startTime) throws IOException {
-        final InetSocketAddress remoteAddress = channel.getRemoteAddress();
-        final Header header = message.getHeader();
-        assert header.needsToReadVariableHeader() == false;
-        ThreadContext threadContext = threadPool.getThreadContext();
-        try (ThreadContext.StoredContext existing = threadContext.stashContext()) {
-            // Place the context with the headers from the message
-            threadContext.setHeaders(header.getHeaders());
-            threadContext.putTransient("_remote_address", remoteAddress);
-            if (header.isRequest()) {
-                handleRequest(channel, header, message);
-            } else {
-                // Responses do not support short circuiting currently
-                assert message.isShortCircuit() == false;
-                final TransportResponseHandler<?> handler;
-                long requestId = header.getRequestId();
-                if (header.isHandshake()) {
-                    handler = handshaker.removeHandlerForHandshake(requestId);
-                } else {
-                    TransportResponseHandler<? extends TransportResponse> theHandler = responseHandlers.onResponseReceived(
-                        requestId,
-                        messageListener
-                    );
-                    if (theHandler == null && header.isError()) {
-                        handler = handshaker.removeHandlerForHandshake(requestId);
-                    } else {
-                        handler = theHandler;
-                    }
-                }
-                // ignore if its null, the service logs it
-                if (handler != null) {
-                    final StreamInput streamInput;
-                    if (message.getContentLength() > 0 || header.getVersion().equals(Version.CURRENT) == false) {
-                        streamInput = namedWriteableStream(message.openOrGetStreamInput());
-                        assertRemoteVersion(streamInput, header.getVersion());
-                        if (header.isError()) {
-                            handlerResponseError(requestId, streamInput, handler);
-                        } else {
-                            handleResponse(requestId, remoteAddress, streamInput, handler);
-                        }
-                    } else {
-                        assert header.isError() == false;
-                        handleResponse(requestId, remoteAddress, EMPTY_STREAM_INPUT, handler);
-                    }
-                }
-
-            }
-        } finally {
-            final long took = threadPool.relativeTimeInMillis() - startTime;
-            final long logThreshold = slowLogThresholdMs;
-            if (logThreshold > 0 && took > logThreshold) {
-                logger.warn(
-                    "handling inbound transport message [{}] took [{}ms] which is above the warn threshold of [{}ms]",
-                    message,
-                    took,
-                    logThreshold
-                );
-            }
-        }
-    }
-
-    private Map<String, Collection<String>> extractHeaders(Map<String, String> headers) {
-        return headers.entrySet().stream().collect(Collectors.toMap(e -> e.getKey(), e -> Collections.singleton(e.getValue())));
-    }
-
-    private <T extends TransportRequest> void handleRequest(TcpChannel channel, Header header, InboundMessage message) throws IOException {
-        final String action = header.getActionName();
-        final long requestId = header.getRequestId();
-        final Version version = header.getVersion();
-        final Map<String, Collection<String>> headers = extractHeaders(header.getHeaders().v1());
-        Span span = tracer.startSpan(SpanBuilder.from(action, channel), headers);
-        try (SpanScope spanScope = tracer.withSpanInScope(span)) {
-            if (header.isHandshake()) {
-                messageListener.onRequestReceived(requestId, action);
-                // Cannot short circuit handshakes
-                assert message.isShortCircuit() == false;
-                final StreamInput stream = namedWriteableStream(message.openOrGetStreamInput());
-                assertRemoteVersion(stream, header.getVersion());
-                final TcpTransportChannel transportChannel = new TcpTransportChannel(
-                    outboundHandler,
-                    channel,
-                    action,
-                    requestId,
-                    version,
-                    header.getFeatures(),
-                    header.isCompressed(),
-                    header.isHandshake(),
-                    message.takeBreakerReleaseControl()
-                );
-                TransportChannel traceableTransportChannel = TraceableTcpTransportChannel.create(transportChannel, span, tracer);
-                try {
-                    handshaker.handleHandshake(traceableTransportChannel, requestId, stream);
-                } catch (Exception e) {
-                    if (Version.CURRENT.isCompatible(header.getVersion())) {
-                        sendErrorResponse(action, traceableTransportChannel, e);
-                    } else {
-                        logger.warn(
-                            new ParameterizedMessage(
-                                "could not send error response to handshake received on [{}] using wire format version [{}], closing channel",
-                                channel,
-                                header.getVersion()
-                            ),
-                            e
-                        );
-                        channel.close();
-                    }
-                }
-            } else {
-                final TcpTransportChannel transportChannel = new TcpTransportChannel(
-                    outboundHandler,
-                    channel,
-                    action,
-                    requestId,
-                    version,
-                    header.getFeatures(),
-                    header.isCompressed(),
-                    header.isHandshake(),
-                    message.takeBreakerReleaseControl()
-                );
-                TransportChannel traceableTransportChannel = TraceableTcpTransportChannel.create(transportChannel, span, tracer);
-                try {
-                    messageListener.onRequestReceived(requestId, action);
-                    if (message.isShortCircuit()) {
-                        sendErrorResponse(action, traceableTransportChannel, message.getException());
-                    } else {
-                        final StreamInput stream = namedWriteableStream(message.openOrGetStreamInput());
-                        assertRemoteVersion(stream, header.getVersion());
-                        final RequestHandlerRegistry<T> reg = requestHandlers.getHandler(action);
-                        assert reg != null;
-
-                        final T request = newRequest(requestId, action, stream, reg);
-                        request.remoteAddress(new TransportAddress(channel.getRemoteAddress()));
-                        checkStreamIsFullyConsumed(requestId, action, stream);
-
-                        final String executor = reg.getExecutor();
-                        if (ThreadPool.Names.SAME.equals(executor)) {
-                            try {
-                                reg.processMessageReceived(request, traceableTransportChannel);
-                            } catch (Exception e) {
-                                sendErrorResponse(reg.getAction(), traceableTransportChannel, e);
-                            }
-                        } else {
-                            threadPool.executor(executor).execute(new RequestHandler<>(reg, request, traceableTransportChannel));
-                        }
-                    }
-                } catch (Exception e) {
-                    sendErrorResponse(action, traceableTransportChannel, e);
-                }
-            }
-        } catch (Exception e) {
-            span.setError(e);
-            span.endSpan();
-            throw e;
-        }
-    }
-
-    /**
-     * Creates new request instance out of input stream. Throws IllegalStateException if the end of
-     * the stream was reached before the request is fully deserialized from the stream.
-     * @param <T> transport request type
-     * @param requestId request identifier
-     * @param action action name
-     * @param stream stream
-     * @param reg request handler registry
-     * @return new request instance
-     * @throws IOException IOException
-     * @throws IllegalStateException IllegalStateException
-     */
-    private <T extends TransportRequest> T newRequest(
-        final long requestId,
-        final String action,
-        final StreamInput stream,
-        final RequestHandlerRegistry<T> reg
-    ) throws IOException {
-        try {
-            return reg.newRequest(stream);
-        } catch (final EOFException e) {
-            // Another favor of (de)serialization issues is when stream contains less bytes than
-            // the request handler needs to deserialize the payload.
-            throw new IllegalStateException(
-                "Message fully read (request) but more data is expected for requestId ["
-                    + requestId
-                    + "], action ["
-                    + action
-                    + "]; resetting",
-                e
-            );
-        }
-    }
-
-    /**
-     * Checks if the stream is fully consumed and throws the exceptions if that is not the case.
-     * @param requestId request identifier
-     * @param action action name
-     * @param stream stream
-     * @throws IOException IOException
-     */
-    private void checkStreamIsFullyConsumed(final long requestId, final String action, final StreamInput stream) throws IOException {
-        // in case we throw an exception, i.e. when the limit is hit, we don't want to verify
-        final int nextByte = stream.read();
-
-        // calling read() is useful to make sure the message is fully read, even if there some kind of EOS marker
-        if (nextByte != -1) {
-            throw new IllegalStateException(
-                "Message not fully read (request) for requestId ["
-                    + requestId
-                    + "], action ["
-                    + action
-                    + "], available ["
-                    + stream.available()
-                    + "]; resetting"
-            );
-        }
-    }
-
-    /**
-     * Checks if the stream is fully consumed and throws the exceptions if that is not the case.
-     * @param requestId request identifier
-     * @param handler response handler
-     * @param stream stream
-     * @param error "true" if response represents error, "false" otherwise
-     * @throws IOException IOException
-     */
-    private void checkStreamIsFullyConsumed(
-        final long requestId,
-        final TransportResponseHandler<?> handler,
-        final StreamInput stream,
-        final boolean error
-    ) throws IOException {
-        if (stream != EMPTY_STREAM_INPUT) {
-            // Check the entire message has been read
-            final int nextByte = stream.read();
-            // calling read() is useful to make sure the message is fully read, even if there is an EOS marker
-            if (nextByte != -1) {
-                throw new IllegalStateException(
-                    "Message not fully read (response) for requestId ["
-                        + requestId
-                        + "], handler ["
-                        + handler
-                        + "], error ["
-                        + error
-                        + "]; resetting"
-                );
-            }
-        }
-    }
-
-    private static void sendErrorResponse(String actionName, TransportChannel transportChannel, Exception e) {
-        try {
-            transportChannel.sendResponse(e);
-        } catch (Exception inner) {
-            inner.addSuppressed(e);
-            logger.warn(() -> new ParameterizedMessage("Failed to send error message back to client for action [{}]", actionName), inner);
-        }
-    }
-
-    private <T extends TransportResponse> void handleResponse(
-        final long requestId,
-        InetSocketAddress remoteAddress,
-        final StreamInput stream,
-        final TransportResponseHandler<T> handler
-    ) {
-        final T response;
-        try {
-            response = handler.read(stream);
-            response.remoteAddress(new TransportAddress(remoteAddress));
-            checkStreamIsFullyConsumed(requestId, handler, stream, false);
-        } catch (Exception e) {
-            final Exception serializationException = new TransportSerializationException(
-                "Failed to deserialize response from handler [" + handler + "]",
-                e
-            );
-            logger.warn(new ParameterizedMessage("Failed to deserialize response from [{}]", remoteAddress), serializationException);
-            handleException(handler, serializationException);
-            return;
-        }
-        final String executor = handler.executor();
-        if (ThreadPool.Names.SAME.equals(executor)) {
-            doHandleResponse(handler, response);
-        } else {
-            threadPool.executor(executor).execute(() -> doHandleResponse(handler, response));
-        }
+        messageReceivedFromPipeline(channel, message, startTime);
     }
 
-    private <T extends TransportResponse> void doHandleResponse(TransportResponseHandler<T> handler, T response) {
-        try {
-            handler.handleResponse(response);
-        } catch (Exception e) {
-            handleException(handler, new ResponseHandlerFailureTransportException(e));
-        }
-    }
-
-    private void handlerResponseError(final long requestId, StreamInput stream, final TransportResponseHandler<?> handler) {
-        Exception error;
-        try {
-            error = stream.readException();
-            checkStreamIsFullyConsumed(requestId, handler, stream, true);
-        } catch (Exception e) {
-            error = new TransportSerializationException(
-                "Failed to deserialize exception response from stream for handler [" + handler + "]",
-                e
-            );
-        }
-        handleException(handler, error);
-    }
-
-    private void handleException(final TransportResponseHandler<?> handler, Throwable error) {
-        if (!(error instanceof RemoteTransportException)) {
-            error = new RemoteTransportException(error.getMessage(), error);
-        }
-        final RemoteTransportException rtx = (RemoteTransportException) error;
-        threadPool.executor(handler.executor()).execute(() -> {
-            try {
-                handler.handleException(rtx);
-            } catch (Exception e) {
-                logger.error(() -> new ParameterizedMessage("failed to handle exception response [{}]", handler), e);
-            }
-        });
-    }
-
-    private StreamInput namedWriteableStream(StreamInput delegate) {
-        return new NamedWriteableAwareStreamInput(delegate, namedWriteableRegistry);
-    }
-
-    static void assertRemoteVersion(StreamInput in, Version version) {
-        assert version.equals(in.getVersion()) : "Stream version [" + in.getVersion() + "] does not match version [" + version + "]";
-    }
-
-    /**
-     * Internal request handler
-     *
-     * @opensearch.internal
-     */
-    private static class RequestHandler<T extends TransportRequest> extends AbstractRunnable {
-        private final RequestHandlerRegistry<T> reg;
-        private final T request;
-        private final TransportChannel transportChannel;
-
-        RequestHandler(RequestHandlerRegistry<T> reg, T request, TransportChannel transportChannel) {
-            this.reg = reg;
-            this.request = request;
-            this.transportChannel = transportChannel;
-        }
-
-        @Override
-        protected void doRun() throws Exception {
-            reg.processMessageReceived(request, transportChannel);
-        }
-
-        @Override
-        public boolean isForceExecution() {
-            return reg.isForceExecution();
-        }
-
-        @Override
-        public void onFailure(Exception e) {
-            sendErrorResponse(reg.getAction(), transportChannel, e);
+    private void messageReceivedFromPipeline(TcpChannel channel, ProtocolInboundMessage message, long startTime) throws IOException {
+        ProtocolMessageHandler protocolMessageHandler = protocolMessageHandlers.get(message.getProtocol());
+        if (protocolMessageHandler == null) {
+            throw new IllegalStateException("No protocol message handler found for protocol: " + message.getProtocol());
         }
+        protocolMessageHandler.messageReceived(channel, message, startTime, slowLogThresholdMs, messageListener);
     }
 }
diff --git a/server/src/main/java/org/opensearch/transport/InboundPipeline.java b/server/src/main/java/org/opensearch/transport/InboundPipeline.java
index dd4690e5e6abf..5cee3bb975223 100644
--- a/server/src/main/java/org/opensearch/transport/InboundPipeline.java
+++ b/server/src/main/java/org/opensearch/transport/InboundPipeline.java
@@ -38,11 +38,11 @@
 import org.opensearch.common.lease.Releasables;
 import org.opensearch.common.util.PageCacheRecycler;
 import org.opensearch.core.common.breaker.CircuitBreaker;
-import org.opensearch.core.common.bytes.CompositeBytesReference;
+import org.opensearch.transport.nativeprotocol.NativeInboundBytesHandler;
 
 import java.io.IOException;
 import java.util.ArrayDeque;
-import java.util.ArrayList;
+import java.util.List;
 import java.util.function.BiConsumer;
 import java.util.function.Function;
 import java.util.function.LongSupplier;
@@ -55,17 +55,16 @@
  */
 public class InboundPipeline implements Releasable {
 
-    private static final ThreadLocal<ArrayList<Object>> fragmentList = ThreadLocal.withInitial(ArrayList::new);
-    private static final InboundMessage PING_MESSAGE = new InboundMessage(null, true);
-
     private final LongSupplier relativeTimeInMillis;
     private final StatsTracker statsTracker;
     private final InboundDecoder decoder;
     private final InboundAggregator aggregator;
-    private final BiConsumer<TcpChannel, InboundMessage> messageHandler;
     private Exception uncaughtException;
     private final ArrayDeque<ReleasableBytesReference> pending = new ArrayDeque<>(2);
     private boolean isClosed = false;
+    private final BiConsumer<TcpChannel, ProtocolInboundMessage> messageHandler;
+    private final List<InboundBytesHandler> protocolBytesHandlers;
+    private InboundBytesHandler currentHandler;
 
     public InboundPipeline(
         Version version,
@@ -74,7 +73,7 @@ public InboundPipeline(
         LongSupplier relativeTimeInMillis,
         Supplier<CircuitBreaker> circuitBreaker,
         Function<String, RequestHandlerRegistry<TransportRequest>> registryFunction,
-        BiConsumer<TcpChannel, InboundMessage> messageHandler
+        BiConsumer<TcpChannel, ProtocolInboundMessage> messageHandler
     ) {
         this(
             statsTracker,
@@ -90,18 +89,23 @@ public InboundPipeline(
         LongSupplier relativeTimeInMillis,
         InboundDecoder decoder,
         InboundAggregator aggregator,
-        BiConsumer<TcpChannel, InboundMessage> messageHandler
+        BiConsumer<TcpChannel, ProtocolInboundMessage> messageHandler
     ) {
         this.relativeTimeInMillis = relativeTimeInMillis;
         this.statsTracker = statsTracker;
         this.decoder = decoder;
         this.aggregator = aggregator;
+        this.protocolBytesHandlers = List.of(new NativeInboundBytesHandler(pending, decoder, aggregator, statsTracker));
         this.messageHandler = messageHandler;
     }
 
     @Override
     public void close() {
         isClosed = true;
+        if (currentHandler != null) {
+            currentHandler.close();
+            currentHandler = null;
+        }
         Releasables.closeWhileHandlingException(decoder, aggregator);
         Releasables.closeWhileHandlingException(pending);
         pending.clear();
@@ -124,95 +128,21 @@ public void doHandleBytes(TcpChannel channel, ReleasableBytesReference reference
         statsTracker.markBytesRead(reference.length());
         pending.add(reference.retain());
 
-        final ArrayList<Object> fragments = fragmentList.get();
-        boolean continueHandling = true;
-
-        while (continueHandling && isClosed == false) {
-            boolean continueDecoding = true;
-            while (continueDecoding && pending.isEmpty() == false) {
-                try (ReleasableBytesReference toDecode = getPendingBytes()) {
-                    final int bytesDecoded = decoder.decode(toDecode, fragments::add);
-                    if (bytesDecoded != 0) {
-                        releasePendingBytes(bytesDecoded);
-                        if (fragments.isEmpty() == false && endOfMessage(fragments.get(fragments.size() - 1))) {
-                            continueDecoding = false;
-                        }
-                    } else {
-                        continueDecoding = false;
-                    }
-                }
-            }
-
-            if (fragments.isEmpty()) {
-                continueHandling = false;
-            } else {
-                try {
-                    forwardFragments(channel, fragments);
-                } finally {
-                    for (Object fragment : fragments) {
-                        if (fragment instanceof ReleasableBytesReference) {
-                            ((ReleasableBytesReference) fragment).close();
-                        }
-                    }
-                    fragments.clear();
-                }
-            }
-        }
-    }
-
-    private void forwardFragments(TcpChannel channel, ArrayList<Object> fragments) throws IOException {
-        for (Object fragment : fragments) {
-            if (fragment instanceof Header) {
-                assert aggregator.isAggregating() == false;
-                aggregator.headerReceived((Header) fragment);
-            } else if (fragment == InboundDecoder.PING) {
-                assert aggregator.isAggregating() == false;
-                messageHandler.accept(channel, PING_MESSAGE);
-            } else if (fragment == InboundDecoder.END_CONTENT) {
-                assert aggregator.isAggregating();
-                try (InboundMessage aggregated = aggregator.finishAggregation()) {
-                    statsTracker.markMessageReceived();
-                    messageHandler.accept(channel, aggregated);
+        // If we don't have a current handler, we should try to find one based on the protocol of the incoming bytes.
+        if (currentHandler == null) {
+            for (InboundBytesHandler handler : protocolBytesHandlers) {
+                if (handler.canHandleBytes(reference)) {
+                    currentHandler = handler;
+                    break;
                 }
-            } else {
-                assert aggregator.isAggregating();
-                assert fragment instanceof ReleasableBytesReference;
-                aggregator.aggregate((ReleasableBytesReference) fragment);
             }
         }
-    }
 
-    private boolean endOfMessage(Object fragment) {
-        return fragment == InboundDecoder.PING || fragment == InboundDecoder.END_CONTENT || fragment instanceof Exception;
-    }
-
-    private ReleasableBytesReference getPendingBytes() {
-        if (pending.size() == 1) {
-            return pending.peekFirst().retain();
+        // If we have a current handler determined based on protocol, we should continue to use it for the fragmented bytes.
+        if (currentHandler != null) {
+            currentHandler.doHandleBytes(channel, reference, messageHandler);
         } else {
-            final ReleasableBytesReference[] bytesReferences = new ReleasableBytesReference[pending.size()];
-            int index = 0;
-            for (ReleasableBytesReference pendingReference : pending) {
-                bytesReferences[index] = pendingReference.retain();
-                ++index;
-            }
-            final Releasable releasable = () -> Releasables.closeWhileHandlingException(bytesReferences);
-            return new ReleasableBytesReference(CompositeBytesReference.of(bytesReferences), releasable);
-        }
-    }
-
-    private void releasePendingBytes(int bytesConsumed) {
-        int bytesToRelease = bytesConsumed;
-        while (bytesToRelease != 0) {
-            try (ReleasableBytesReference reference = pending.pollFirst()) {
-                assert reference != null;
-                if (bytesToRelease < reference.length()) {
-                    pending.addFirst(reference.retainedSlice(bytesToRelease, reference.length() - bytesToRelease));
-                    bytesToRelease -= bytesToRelease;
-                } else {
-                    bytesToRelease -= reference.length();
-                }
-            }
+            throw new IllegalStateException("No bytes handler found for the incoming transport protocol");
         }
     }
 }
diff --git a/server/src/main/java/org/opensearch/transport/NativeMessageHandler.java b/server/src/main/java/org/opensearch/transport/NativeMessageHandler.java
new file mode 100644
index 0000000000000..4c972fdc14fa5
--- /dev/null
+++ b/server/src/main/java/org/opensearch/transport/NativeMessageHandler.java
@@ -0,0 +1,507 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+package org.opensearch.transport;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.apache.lucene.util.BytesRef;
+import org.opensearch.Version;
+import org.opensearch.common.util.BigArrays;
+import org.opensearch.common.util.concurrent.AbstractRunnable;
+import org.opensearch.common.util.concurrent.ThreadContext;
+import org.opensearch.core.common.io.stream.ByteBufferStreamInput;
+import org.opensearch.core.common.io.stream.NamedWriteableAwareStreamInput;
+import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
+import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.transport.TransportAddress;
+import org.opensearch.core.transport.TransportResponse;
+import org.opensearch.telemetry.tracing.Span;
+import org.opensearch.telemetry.tracing.SpanBuilder;
+import org.opensearch.telemetry.tracing.SpanScope;
+import org.opensearch.telemetry.tracing.Tracer;
+import org.opensearch.telemetry.tracing.channels.TraceableTcpTransportChannel;
+import org.opensearch.threadpool.ThreadPool;
+import org.opensearch.transport.nativeprotocol.NativeInboundMessage;
+import org.opensearch.transport.nativeprotocol.NativeOutboundHandler;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * Native handler for inbound data
+ *
+ * @opensearch.internal
+ */
+public class NativeMessageHandler implements ProtocolMessageHandler {
+
+    private static final Logger logger = LogManager.getLogger(NativeMessageHandler.class);
+
+    private final ThreadPool threadPool;
+    private final NativeOutboundHandler outboundHandler;
+    private final NamedWriteableRegistry namedWriteableRegistry;
+    private final TransportHandshaker handshaker;
+    private final TransportKeepAlive keepAlive;
+    private final Transport.ResponseHandlers responseHandlers;
+    private final Transport.RequestHandlers requestHandlers;
+
+    private final Tracer tracer;
+
+    NativeMessageHandler(
+        String nodeName,
+        Version version,
+        String[] features,
+        StatsTracker statsTracker,
+        ThreadPool threadPool,
+        BigArrays bigArrays,
+        OutboundHandler outboundHandler,
+        NamedWriteableRegistry namedWriteableRegistry,
+        TransportHandshaker handshaker,
+        Transport.RequestHandlers requestHandlers,
+        Transport.ResponseHandlers responseHandlers,
+        Tracer tracer,
+        TransportKeepAlive keepAlive
+    ) {
+        this.threadPool = threadPool;
+        this.outboundHandler = new NativeOutboundHandler(nodeName, version, features, statsTracker, threadPool, bigArrays, outboundHandler);
+        this.namedWriteableRegistry = namedWriteableRegistry;
+        this.handshaker = handshaker;
+        this.requestHandlers = requestHandlers;
+        this.responseHandlers = responseHandlers;
+        this.tracer = tracer;
+        this.keepAlive = keepAlive;
+    }
+
+    // Empty stream constant to avoid instantiating a new stream for empty messages.
+    private static final StreamInput EMPTY_STREAM_INPUT = new ByteBufferStreamInput(ByteBuffer.wrap(BytesRef.EMPTY_BYTES));
+
+    @Override
+    public void messageReceived(
+        TcpChannel channel,
+        ProtocolInboundMessage message,
+        long startTime,
+        long slowLogThresholdMs,
+        TransportMessageListener messageListener
+    ) throws IOException {
+        NativeInboundMessage inboundMessage = (NativeInboundMessage) message;
+        TransportLogger.logInboundMessage(channel, inboundMessage);
+        if (inboundMessage.isPing()) {
+            keepAlive.receiveKeepAlive(channel);
+        } else {
+            handleMessage(channel, inboundMessage, startTime, slowLogThresholdMs, messageListener);
+        }
+    }
+
+    private void handleMessage(
+        TcpChannel channel,
+        NativeInboundMessage message,
+        long startTime,
+        long slowLogThresholdMs,
+        TransportMessageListener messageListener
+    ) throws IOException {
+        final InetSocketAddress remoteAddress = channel.getRemoteAddress();
+        final Header header = message.getHeader();
+        assert header.needsToReadVariableHeader() == false;
+        ThreadContext threadContext = threadPool.getThreadContext();
+        try (ThreadContext.StoredContext existing = threadContext.stashContext()) {
+            // Place the context with the headers from the message
+            threadContext.setHeaders(header.getHeaders());
+            threadContext.putTransient("_remote_address", remoteAddress);
+            if (header.isRequest()) {
+                handleRequest(channel, header, message, messageListener);
+            } else {
+                // Responses do not support short circuiting currently
+                assert message.isShortCircuit() == false;
+                final TransportResponseHandler<?> handler;
+                long requestId = header.getRequestId();
+                if (header.isHandshake()) {
+                    handler = handshaker.removeHandlerForHandshake(requestId);
+                } else {
+                    TransportResponseHandler<? extends TransportResponse> theHandler = responseHandlers.onResponseReceived(
+                        requestId,
+                        messageListener
+                    );
+                    if (theHandler == null && header.isError()) {
+                        handler = handshaker.removeHandlerForHandshake(requestId);
+                    } else {
+                        handler = theHandler;
+                    }
+                }
+                // ignore if its null, the service logs it
+                if (handler != null) {
+                    final StreamInput streamInput;
+                    if (message.getContentLength() > 0 || header.getVersion().equals(Version.CURRENT) == false) {
+                        streamInput = namedWriteableStream(message.openOrGetStreamInput());
+                        assertRemoteVersion(streamInput, header.getVersion());
+                        if (header.isError()) {
+                            handlerResponseError(requestId, streamInput, handler);
+                        } else {
+                            handleResponse(requestId, remoteAddress, streamInput, handler);
+                        }
+                    } else {
+                        assert header.isError() == false;
+                        handleResponse(requestId, remoteAddress, EMPTY_STREAM_INPUT, handler);
+                    }
+                }
+
+            }
+        } finally {
+            final long took = threadPool.relativeTimeInMillis() - startTime;
+            final long logThreshold = slowLogThresholdMs;
+            if (logThreshold > 0 && took > logThreshold) {
+                logger.warn(
+                    "handling inbound transport message [{}] took [{}ms] which is above the warn threshold of [{}ms]",
+                    message,
+                    took,
+                    logThreshold
+                );
+            }
+        }
+    }
+
+    private Map<String, Collection<String>> extractHeaders(Map<String, String> headers) {
+        return headers.entrySet().stream().collect(Collectors.toMap(e -> e.getKey(), e -> Collections.singleton(e.getValue())));
+    }
+
+    private <T extends TransportRequest> void handleRequest(
+        TcpChannel channel,
+        Header header,
+        NativeInboundMessage message,
+        TransportMessageListener messageListener
+    ) throws IOException {
+        final String action = header.getActionName();
+        final long requestId = header.getRequestId();
+        final Version version = header.getVersion();
+        final Map<String, Collection<String>> headers = extractHeaders(header.getHeaders().v1());
+        Span span = tracer.startSpan(SpanBuilder.from(action, channel), headers);
+        try (SpanScope spanScope = tracer.withSpanInScope(span)) {
+            if (header.isHandshake()) {
+                messageListener.onRequestReceived(requestId, action);
+                // Cannot short circuit handshakes
+                assert message.isShortCircuit() == false;
+                final StreamInput stream = namedWriteableStream(message.openOrGetStreamInput());
+                assertRemoteVersion(stream, header.getVersion());
+                final TcpTransportChannel transportChannel = new TcpTransportChannel(
+                    outboundHandler,
+                    channel,
+                    action,
+                    requestId,
+                    version,
+                    header.getFeatures(),
+                    header.isCompressed(),
+                    header.isHandshake(),
+                    message.takeBreakerReleaseControl()
+                );
+                TransportChannel traceableTransportChannel = TraceableTcpTransportChannel.create(transportChannel, span, tracer);
+                try {
+                    handshaker.handleHandshake(traceableTransportChannel, requestId, stream);
+                } catch (Exception e) {
+                    if (Version.CURRENT.isCompatible(header.getVersion())) {
+                        sendErrorResponse(action, traceableTransportChannel, e);
+                    } else {
+                        logger.warn(
+                            new ParameterizedMessage(
+                                "could not send error response to handshake received on [{}] using wire format version [{}], closing channel",
+                                channel,
+                                header.getVersion()
+                            ),
+                            e
+                        );
+                        channel.close();
+                    }
+                }
+            } else {
+                final TcpTransportChannel transportChannel = new TcpTransportChannel(
+                    outboundHandler,
+                    channel,
+                    action,
+                    requestId,
+                    version,
+                    header.getFeatures(),
+                    header.isCompressed(),
+                    header.isHandshake(),
+                    message.takeBreakerReleaseControl()
+                );
+                TransportChannel traceableTransportChannel = TraceableTcpTransportChannel.create(transportChannel, span, tracer);
+                try {
+                    messageListener.onRequestReceived(requestId, action);
+                    if (message.isShortCircuit()) {
+                        sendErrorResponse(action, traceableTransportChannel, message.getException());
+                    } else {
+                        final StreamInput stream = namedWriteableStream(message.openOrGetStreamInput());
+                        assertRemoteVersion(stream, header.getVersion());
+                        final RequestHandlerRegistry<T> reg = requestHandlers.getHandler(action);
+                        assert reg != null;
+
+                        final T request = newRequest(requestId, action, stream, reg);
+                        request.remoteAddress(new TransportAddress(channel.getRemoteAddress()));
+                        checkStreamIsFullyConsumed(requestId, action, stream);
+
+                        final String executor = reg.getExecutor();
+                        if (ThreadPool.Names.SAME.equals(executor)) {
+                            try {
+                                reg.processMessageReceived(request, traceableTransportChannel);
+                            } catch (Exception e) {
+                                sendErrorResponse(reg.getAction(), traceableTransportChannel, e);
+                            }
+                        } else {
+                            threadPool.executor(executor).execute(new RequestHandler<>(reg, request, traceableTransportChannel));
+                        }
+                    }
+                } catch (Exception e) {
+                    sendErrorResponse(action, traceableTransportChannel, e);
+                }
+            }
+        } catch (Exception e) {
+            span.setError(e);
+            span.endSpan();
+            throw e;
+        }
+    }
+
+    /**
+     * Creates new request instance out of input stream. Throws IllegalStateException if the end of
+     * the stream was reached before the request is fully deserialized from the stream.
+     * @param <T> transport request type
+     * @param requestId request identifier
+     * @param action action name
+     * @param stream stream
+     * @param reg request handler registry
+     * @return new request instance
+     * @throws IOException IOException
+     * @throws IllegalStateException IllegalStateException
+     */
+    private <T extends TransportRequest> T newRequest(
+        final long requestId,
+        final String action,
+        final StreamInput stream,
+        final RequestHandlerRegistry<T> reg
+    ) throws IOException {
+        try {
+            return reg.newRequest(stream);
+        } catch (final EOFException e) {
+            // Another favor of (de)serialization issues is when stream contains less bytes than
+            // the request handler needs to deserialize the payload.
+            throw new IllegalStateException(
+                "Message fully read (request) but more data is expected for requestId ["
+                    + requestId
+                    + "], action ["
+                    + action
+                    + "]; resetting",
+                e
+            );
+        }
+    }
+
+    /**
+     * Checks if the stream is fully consumed and throws the exceptions if that is not the case.
+     * @param requestId request identifier
+     * @param action action name
+     * @param stream stream
+     * @throws IOException IOException
+     */
+    private void checkStreamIsFullyConsumed(final long requestId, final String action, final StreamInput stream) throws IOException {
+        // in case we throw an exception, i.e. when the limit is hit, we don't want to verify
+        final int nextByte = stream.read();
+
+        // calling read() is useful to make sure the message is fully read, even if there some kind of EOS marker
+        if (nextByte != -1) {
+            throw new IllegalStateException(
+                "Message not fully read (request) for requestId ["
+                    + requestId
+                    + "], action ["
+                    + action
+                    + "], available ["
+                    + stream.available()
+                    + "]; resetting"
+            );
+        }
+    }
+
+    /**
+     * Checks if the stream is fully consumed and throws the exceptions if that is not the case.
+     * @param requestId request identifier
+     * @param handler response handler
+     * @param stream stream
+     * @param error "true" if response represents error, "false" otherwise
+     * @throws IOException IOException
+     */
+    private void checkStreamIsFullyConsumed(
+        final long requestId,
+        final TransportResponseHandler<?> handler,
+        final StreamInput stream,
+        final boolean error
+    ) throws IOException {
+        if (stream != EMPTY_STREAM_INPUT) {
+            // Check the entire message has been read
+            final int nextByte = stream.read();
+            // calling read() is useful to make sure the message is fully read, even if there is an EOS marker
+            if (nextByte != -1) {
+                throw new IllegalStateException(
+                    "Message not fully read (response) for requestId ["
+                        + requestId
+                        + "], handler ["
+                        + handler
+                        + "], error ["
+                        + error
+                        + "]; resetting"
+                );
+            }
+        }
+    }
+
+    private static void sendErrorResponse(String actionName, TransportChannel transportChannel, Exception e) {
+        try {
+            transportChannel.sendResponse(e);
+        } catch (Exception inner) {
+            inner.addSuppressed(e);
+            logger.warn(() -> new ParameterizedMessage("Failed to send error message back to client for action [{}]", actionName), inner);
+        }
+    }
+
+    private <T extends TransportResponse> void handleResponse(
+        final long requestId,
+        InetSocketAddress remoteAddress,
+        final StreamInput stream,
+        final TransportResponseHandler<T> handler
+    ) {
+        final T response;
+        try {
+            response = handler.read(stream);
+            response.remoteAddress(new TransportAddress(remoteAddress));
+            checkStreamIsFullyConsumed(requestId, handler, stream, false);
+        } catch (Exception e) {
+            final Exception serializationException = new TransportSerializationException(
+                "Failed to deserialize response from handler [" + handler + "]",
+                e
+            );
+            logger.warn(new ParameterizedMessage("Failed to deserialize response from [{}]", remoteAddress), serializationException);
+            handleException(handler, serializationException);
+            return;
+        }
+        final String executor = handler.executor();
+        if (ThreadPool.Names.SAME.equals(executor)) {
+            doHandleResponse(handler, response);
+        } else {
+            threadPool.executor(executor).execute(() -> doHandleResponse(handler, response));
+        }
+    }
+
+    private <T extends TransportResponse> void doHandleResponse(TransportResponseHandler<T> handler, T response) {
+        try {
+            handler.handleResponse(response);
+        } catch (Exception e) {
+            handleException(handler, new ResponseHandlerFailureTransportException(e));
+        }
+    }
+
+    private void handlerResponseError(final long requestId, StreamInput stream, final TransportResponseHandler<?> handler) {
+        Exception error;
+        try {
+            error = stream.readException();
+            checkStreamIsFullyConsumed(requestId, handler, stream, true);
+        } catch (Exception e) {
+            error = new TransportSerializationException(
+                "Failed to deserialize exception response from stream for handler [" + handler + "]",
+                e
+            );
+        }
+        handleException(handler, error);
+    }
+
+    private void handleException(final TransportResponseHandler<?> handler, Throwable error) {
+        if (!(error instanceof RemoteTransportException)) {
+            error = new RemoteTransportException(error.getMessage(), error);
+        }
+        final RemoteTransportException rtx = (RemoteTransportException) error;
+        threadPool.executor(handler.executor()).execute(() -> {
+            try {
+                handler.handleException(rtx);
+            } catch (Exception e) {
+                logger.error(() -> new ParameterizedMessage("failed to handle exception response [{}]", handler), e);
+            }
+        });
+    }
+
+    private StreamInput namedWriteableStream(StreamInput delegate) {
+        return new NamedWriteableAwareStreamInput(delegate, namedWriteableRegistry);
+    }
+
+    static void assertRemoteVersion(StreamInput in, Version version) {
+        assert version.equals(in.getVersion()) : "Stream version [" + in.getVersion() + "] does not match version [" + version + "]";
+    }
+
+    /**
+     * Internal request handler
+     *
+     * @opensearch.internal
+     */
+    private static class RequestHandler<T extends TransportRequest> extends AbstractRunnable {
+        private final RequestHandlerRegistry<T> reg;
+        private final T request;
+        private final TransportChannel transportChannel;
+
+        RequestHandler(RequestHandlerRegistry<T> reg, T request, TransportChannel transportChannel) {
+            this.reg = reg;
+            this.request = request;
+            this.transportChannel = transportChannel;
+        }
+
+        @Override
+        protected void doRun() throws Exception {
+            reg.processMessageReceived(request, transportChannel);
+        }
+
+        @Override
+        public boolean isForceExecution() {
+            return reg.isForceExecution();
+        }
+
+        @Override
+        public void onFailure(Exception e) {
+            sendErrorResponse(reg.getAction(), transportChannel, e);
+        }
+    }
+
+    @Override
+    public void setMessageListener(TransportMessageListener listener) {
+        outboundHandler.setMessageListener(listener);
+    }
+
+}
diff --git a/server/src/main/java/org/opensearch/transport/OutboundHandler.java b/server/src/main/java/org/opensearch/transport/OutboundHandler.java
index b83dbdd0effe4..43f53e4011260 100644
--- a/server/src/main/java/org/opensearch/transport/OutboundHandler.java
+++ b/server/src/main/java/org/opensearch/transport/OutboundHandler.java
@@ -35,164 +35,47 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
-import org.opensearch.Version;
-import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.common.CheckedSupplier;
-import org.opensearch.common.io.stream.ReleasableBytesStreamOutput;
 import org.opensearch.common.lease.Releasable;
 import org.opensearch.common.lease.Releasables;
 import org.opensearch.common.network.CloseableChannel;
 import org.opensearch.common.transport.NetworkExceptionHelper;
-import org.opensearch.common.util.BigArrays;
 import org.opensearch.common.util.concurrent.ThreadContext;
-import org.opensearch.common.util.io.IOUtils;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.action.NotifyOnceListener;
 import org.opensearch.core.common.bytes.BytesReference;
-import org.opensearch.core.common.transport.TransportAddress;
-import org.opensearch.core.transport.TransportResponse;
 import org.opensearch.threadpool.ThreadPool;
 
 import java.io.IOException;
-import java.util.Set;
 
 /**
  * Outbound data handler
  *
  * @opensearch.internal
  */
-final class OutboundHandler {
+public final class OutboundHandler {
 
     private static final Logger logger = LogManager.getLogger(OutboundHandler.class);
 
-    private final String nodeName;
-    private final Version version;
-    private final String[] features;
     private final StatsTracker statsTracker;
     private final ThreadPool threadPool;
-    private final BigArrays bigArrays;
-    private volatile TransportMessageListener messageListener = TransportMessageListener.NOOP_LISTENER;
 
-    OutboundHandler(
-        String nodeName,
-        Version version,
-        String[] features,
-        StatsTracker statsTracker,
-        ThreadPool threadPool,
-        BigArrays bigArrays
-    ) {
-        this.nodeName = nodeName;
-        this.version = version;
-        this.features = features;
+    public OutboundHandler(StatsTracker statsTracker, ThreadPool threadPool) {
         this.statsTracker = statsTracker;
         this.threadPool = threadPool;
-        this.bigArrays = bigArrays;
     }
 
     void sendBytes(TcpChannel channel, BytesReference bytes, ActionListener<Void> listener) {
-        SendContext sendContext = new SendContext(channel, () -> bytes, listener);
+        SendContext sendContext = new SendContext(statsTracker, channel, () -> bytes, listener);
         try {
-            internalSend(channel, sendContext);
+            sendBytes(channel, sendContext);
         } catch (IOException e) {
             // This should not happen as the bytes are already serialized
             throw new AssertionError(e);
         }
     }
 
-    /**
-     * Sends the request to the given channel. This method should be used to send {@link TransportRequest}
-     * objects back to the caller.
-     */
-    void sendRequest(
-        final DiscoveryNode node,
-        final TcpChannel channel,
-        final long requestId,
-        final String action,
-        final TransportRequest request,
-        final TransportRequestOptions options,
-        final Version channelVersion,
-        final boolean compressRequest,
-        final boolean isHandshake
-    ) throws IOException, TransportException {
-        Version version = Version.min(this.version, channelVersion);
-        OutboundMessage.Request message = new OutboundMessage.Request(
-            threadPool.getThreadContext(),
-            features,
-            request,
-            version,
-            action,
-            requestId,
-            isHandshake,
-            compressRequest
-        );
-        ActionListener<Void> listener = ActionListener.wrap(() -> messageListener.onRequestSent(node, requestId, action, request, options));
-        sendMessage(channel, message, listener);
-    }
-
-    /**
-     * Sends the response to the given channel. This method should be used to send {@link TransportResponse}
-     * objects back to the caller.
-     *
-     * @see #sendErrorResponse(Version, Set, TcpChannel, long, String, Exception) for sending error responses
-     */
-    void sendResponse(
-        final Version nodeVersion,
-        final Set<String> features,
-        final TcpChannel channel,
-        final long requestId,
-        final String action,
-        final TransportResponse response,
-        final boolean compress,
-        final boolean isHandshake
-    ) throws IOException {
-        Version version = Version.min(this.version, nodeVersion);
-        OutboundMessage.Response message = new OutboundMessage.Response(
-            threadPool.getThreadContext(),
-            features,
-            response,
-            version,
-            requestId,
-            isHandshake,
-            compress
-        );
-        ActionListener<Void> listener = ActionListener.wrap(() -> messageListener.onResponseSent(requestId, action, response));
-        sendMessage(channel, message, listener);
-    }
-
-    /**
-     * Sends back an error response to the caller via the given channel
-     */
-    void sendErrorResponse(
-        final Version nodeVersion,
-        final Set<String> features,
-        final TcpChannel channel,
-        final long requestId,
-        final String action,
-        final Exception error
-    ) throws IOException {
-        Version version = Version.min(this.version, nodeVersion);
-        TransportAddress address = new TransportAddress(channel.getLocalAddress());
-        RemoteTransportException tx = new RemoteTransportException(nodeName, address, action, error);
-        OutboundMessage.Response message = new OutboundMessage.Response(
-            threadPool.getThreadContext(),
-            features,
-            tx,
-            version,
-            requestId,
-            false,
-            false
-        );
-        ActionListener<Void> listener = ActionListener.wrap(() -> messageListener.onResponseSent(requestId, action, error));
-        sendMessage(channel, message, listener);
-    }
-
-    private void sendMessage(TcpChannel channel, OutboundMessage networkMessage, ActionListener<Void> listener) throws IOException {
-        MessageSerializer serializer = new MessageSerializer(networkMessage, bigArrays);
-        SendContext sendContext = new SendContext(channel, serializer, listener, serializer);
-        internalSend(channel, sendContext);
-    }
-
-    private void internalSend(TcpChannel channel, SendContext sendContext) throws IOException {
+    public void sendBytes(TcpChannel channel, SendContext sendContext) throws IOException {
         channel.getChannelStats().markAccessed(threadPool.relativeTimeInMillis());
         BytesReference reference = sendContext.get();
         // stash thread context so that channel event loop is not polluted by thread context
@@ -205,59 +88,30 @@ private void internalSend(TcpChannel channel, SendContext sendContext) throws IO
         }
     }
 
-    void setMessageListener(TransportMessageListener listener) {
-        if (messageListener == TransportMessageListener.NOOP_LISTENER) {
-            messageListener = listener;
-        } else {
-            throw new IllegalStateException("Cannot set message listener twice");
-        }
-    }
-
     /**
      * Internal message serializer
      *
      * @opensearch.internal
      */
-    private static class MessageSerializer implements CheckedSupplier<BytesReference, IOException>, Releasable {
-
-        private final OutboundMessage message;
-        private final BigArrays bigArrays;
-        private volatile ReleasableBytesStreamOutput bytesStreamOutput;
-
-        private MessageSerializer(OutboundMessage message, BigArrays bigArrays) {
-            this.message = message;
-            this.bigArrays = bigArrays;
-        }
-
-        @Override
-        public BytesReference get() throws IOException {
-            bytesStreamOutput = new ReleasableBytesStreamOutput(bigArrays);
-            return message.serialize(bytesStreamOutput);
-        }
-
-        @Override
-        public void close() {
-            IOUtils.closeWhileHandlingException(bytesStreamOutput);
-        }
-    }
-
-    private class SendContext extends NotifyOnceListener<Void> implements CheckedSupplier<BytesReference, IOException> {
-
+    public static class SendContext extends NotifyOnceListener<Void> implements CheckedSupplier<BytesReference, IOException> {
+        private final StatsTracker statsTracker;
         private final TcpChannel channel;
         private final CheckedSupplier<BytesReference, IOException> messageSupplier;
         private final ActionListener<Void> listener;
         private final Releasable optionalReleasable;
         private long messageSize = -1;
 
-        private SendContext(
+        SendContext(
+            StatsTracker statsTracker,
             TcpChannel channel,
             CheckedSupplier<BytesReference, IOException> messageSupplier,
             ActionListener<Void> listener
         ) {
-            this(channel, messageSupplier, listener, null);
+            this(statsTracker, channel, messageSupplier, listener, null);
         }
 
-        private SendContext(
+        public SendContext(
+            StatsTracker statsTracker,
             TcpChannel channel,
             CheckedSupplier<BytesReference, IOException> messageSupplier,
             ActionListener<Void> listener,
@@ -267,6 +121,7 @@ private SendContext(
             this.messageSupplier = messageSupplier;
             this.listener = listener;
             this.optionalReleasable = optionalReleasable;
+            this.statsTracker = statsTracker;
         }
 
         public BytesReference get() throws IOException {
diff --git a/server/src/main/java/org/opensearch/transport/ProtocolInboundMessage.java b/server/src/main/java/org/opensearch/transport/ProtocolInboundMessage.java
new file mode 100644
index 0000000000000..43c2d5ffe4c96
--- /dev/null
+++ b/server/src/main/java/org/opensearch/transport/ProtocolInboundMessage.java
@@ -0,0 +1,27 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.transport;
+
+import org.opensearch.common.annotation.PublicApi;
+
+/**
+ * Base class for inbound data as a message.
+ * Different implementations are used for different protocols.
+ *
+ * @opensearch.internal
+ */
+@PublicApi(since = "2.14.0")
+public interface ProtocolInboundMessage {
+
+    /**
+     * @return the protocol used to encode this message
+     */
+    public String getProtocol();
+
+}
diff --git a/server/src/main/java/org/opensearch/transport/ProtocolMessageHandler.java b/server/src/main/java/org/opensearch/transport/ProtocolMessageHandler.java
new file mode 100644
index 0000000000000..3c3fafebc34df
--- /dev/null
+++ b/server/src/main/java/org/opensearch/transport/ProtocolMessageHandler.java
@@ -0,0 +1,41 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.transport;
+
+import java.io.IOException;
+
+/**
+ * Interface for message handlers based on transport protocol.
+ *
+ * @opensearch.internal
+ */
+public interface ProtocolMessageHandler {
+
+    /**
+     * Handles the message received on the channel.
+     * @param channel the channel on which the message was received
+     * @param message the message received
+     * @param startTime the start time
+     * @param slowLogThresholdMs the threshold for slow logs
+     * @param messageListener the message listener
+     */
+    public void messageReceived(
+        TcpChannel channel,
+        ProtocolInboundMessage message,
+        long startTime,
+        long slowLogThresholdMs,
+        TransportMessageListener messageListener
+    ) throws IOException;
+
+    /**
+     * Sets the message listener to be used by the handler.
+     * @param listener the message listener
+     */
+    public void setMessageListener(TransportMessageListener listener);
+}
diff --git a/server/src/main/java/org/opensearch/transport/ProtocolOutboundHandler.java b/server/src/main/java/org/opensearch/transport/ProtocolOutboundHandler.java
new file mode 100644
index 0000000000000..42c5462fddf80
--- /dev/null
+++ b/server/src/main/java/org/opensearch/transport/ProtocolOutboundHandler.java
@@ -0,0 +1,70 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.transport;
+
+import org.opensearch.Version;
+import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.core.transport.TransportResponse;
+
+import java.io.IOException;
+import java.util.Set;
+
+/**
+ * Protocol based outbound data handler.
+ * Different transport protocols can have different implementations of this class.
+ *
+ * @opensearch.internal
+ */
+public abstract class ProtocolOutboundHandler {
+
+    /**
+     * Sends the request to the given channel. This method should be used to send {@link TransportRequest}
+     * objects back to the caller.
+     */
+    public abstract void sendRequest(
+        final DiscoveryNode node,
+        final TcpChannel channel,
+        final long requestId,
+        final String action,
+        final TransportRequest request,
+        final TransportRequestOptions options,
+        final Version channelVersion,
+        final boolean compressRequest,
+        final boolean isHandshake
+    ) throws IOException, TransportException;
+
+    /**
+     * Sends the response to the given channel. This method should be used to send {@link TransportResponse}
+     * objects back to the caller.
+     *
+     * @see #sendErrorResponse(Version, Set, TcpChannel, long, String, Exception) for sending error responses
+     */
+    public abstract void sendResponse(
+        final Version nodeVersion,
+        final Set<String> features,
+        final TcpChannel channel,
+        final long requestId,
+        final String action,
+        final TransportResponse response,
+        final boolean compress,
+        final boolean isHandshake
+    ) throws IOException;
+
+    /**
+     * Sends back an error response to the caller via the given channel
+     */
+    public abstract void sendErrorResponse(
+        final Version nodeVersion,
+        final Set<String> features,
+        final TcpChannel channel,
+        final long requestId,
+        final String action,
+        final Exception error
+    ) throws IOException;
+}
diff --git a/server/src/main/java/org/opensearch/transport/RemoteClusterService.java b/server/src/main/java/org/opensearch/transport/RemoteClusterService.java
index 87786fb22f22e..dc729053c2148 100644
--- a/server/src/main/java/org/opensearch/transport/RemoteClusterService.java
+++ b/server/src/main/java/org/opensearch/transport/RemoteClusterService.java
@@ -40,6 +40,7 @@
 import org.opensearch.client.Client;
 import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.cluster.node.DiscoveryNodeRole;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
@@ -73,8 +74,9 @@
 /**
  * Basic service for accessing remote clusters via gateway nodes
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class RemoteClusterService extends RemoteClusterAware implements Closeable {
 
     private final Logger logger = LogManager.getLogger(RemoteClusterService.class);
diff --git a/server/src/main/java/org/opensearch/transport/RemoteConnectionInfo.java b/server/src/main/java/org/opensearch/transport/RemoteConnectionInfo.java
index 280dd958358fd..b5f356490d3d4 100644
--- a/server/src/main/java/org/opensearch/transport/RemoteConnectionInfo.java
+++ b/server/src/main/java/org/opensearch/transport/RemoteConnectionInfo.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.transport;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
@@ -46,8 +47,9 @@
  * This class encapsulates all remote cluster information to be rendered on
  * {@code _remote/info} requests.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class RemoteConnectionInfo implements ToXContentFragment, Writeable {
 
     final ModeInfo modeInfo;
@@ -132,8 +134,9 @@ public int hashCode() {
     /**
      * Mode information
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public interface ModeInfo extends ToXContentFragment, Writeable {
 
         boolean isConnected();
diff --git a/server/src/main/java/org/opensearch/transport/RemoteConnectionStrategy.java b/server/src/main/java/org/opensearch/transport/RemoteConnectionStrategy.java
index f0b37c617725e..f2c159d1380e8 100644
--- a/server/src/main/java/org/opensearch/transport/RemoteConnectionStrategy.java
+++ b/server/src/main/java/org/opensearch/transport/RemoteConnectionStrategy.java
@@ -38,6 +38,7 @@
 import org.apache.lucene.store.AlreadyClosedException;
 import org.opensearch.action.support.ContextPreservingActionListener;
 import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
@@ -75,7 +76,13 @@
  */
 public abstract class RemoteConnectionStrategy implements TransportConnectionListener, Closeable {
 
-    enum ConnectionStrategy {
+    /**
+     * Strategy to connect to remote nodes
+     *
+    * @opensearch.api
+    */
+    @PublicApi(since = "1.0.0")
+    public enum ConnectionStrategy {
         SNIFF(
             SniffConnectionStrategy.CHANNELS_PER_CONNECTION,
             SniffConnectionStrategy::enablementSettings,
diff --git a/server/src/main/java/org/opensearch/transport/RequestHandlerRegistry.java b/server/src/main/java/org/opensearch/transport/RequestHandlerRegistry.java
index 98c182c562928..4368dbdece6cf 100644
--- a/server/src/main/java/org/opensearch/transport/RequestHandlerRegistry.java
+++ b/server/src/main/java/org/opensearch/transport/RequestHandlerRegistry.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.transport;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.lease.Releasable;
 import org.opensearch.common.lease.Releasables;
 import org.opensearch.common.util.concurrent.ThreadContext;
@@ -47,8 +48,9 @@
 /**
  * Registry for OpenSearch RequestHandlers
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class RequestHandlerRegistry<Request extends TransportRequest> {
 
     private final String action;
diff --git a/server/src/main/java/org/opensearch/transport/StatsTracker.java b/server/src/main/java/org/opensearch/transport/StatsTracker.java
index 5548d2d558ae2..02bc0a51c9330 100644
--- a/server/src/main/java/org/opensearch/transport/StatsTracker.java
+++ b/server/src/main/java/org/opensearch/transport/StatsTracker.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.transport;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.metrics.MeanMetric;
 
 import java.util.concurrent.atomic.LongAdder;
@@ -39,8 +40,9 @@
 /**
  * Tracks transport statistics
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public final class StatsTracker {
 
     private final LongAdder bytesRead = new LongAdder();
diff --git a/server/src/main/java/org/opensearch/transport/TcpChannel.java b/server/src/main/java/org/opensearch/transport/TcpChannel.java
index f98b65d0a4df1..7d4515de85d80 100644
--- a/server/src/main/java/org/opensearch/transport/TcpChannel.java
+++ b/server/src/main/java/org/opensearch/transport/TcpChannel.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.transport;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.network.CloseableChannel;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.action.ActionListener;
@@ -45,8 +46,9 @@
  * abstraction used by the {@link TcpTransport} and {@link TransportService}. All tcp transport
  * implementations must return channels that adhere to the required method contracts.
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface TcpChannel extends CloseableChannel {
 
     /**
@@ -114,8 +116,9 @@ default <T> Optional<T> get(String name, Class<T> clazz) {
     /**
      * Channel statistics
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     class ChannelStats {
 
         private volatile long lastAccessedTime;
diff --git a/server/src/main/java/org/opensearch/transport/TcpTransport.java b/server/src/main/java/org/opensearch/transport/TcpTransport.java
index d0e6516973382..ffa3168da0b3e 100644
--- a/server/src/main/java/org/opensearch/transport/TcpTransport.java
+++ b/server/src/main/java/org/opensearch/transport/TcpTransport.java
@@ -39,6 +39,7 @@
 import org.opensearch.action.support.ThreadedActionListener;
 import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.common.Booleans;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.lifecycle.AbstractLifecycleComponent;
 import org.opensearch.common.lifecycle.Lifecycle;
 import org.opensearch.common.metrics.MeanMetric;
@@ -70,6 +71,7 @@
 import org.opensearch.node.Node;
 import org.opensearch.telemetry.tracing.Tracer;
 import org.opensearch.threadpool.ThreadPool;
+import org.opensearch.transport.nativeprotocol.NativeOutboundHandler;
 
 import java.io.IOException;
 import java.io.StreamCorruptedException;
@@ -111,8 +113,9 @@
 /**
  * The TCP Transport layer
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public abstract class TcpTransport extends AbstractLifecycleComponent implements Transport {
     private static final Logger logger = LogManager.getLogger(TcpTransport.class);
 
@@ -148,6 +151,7 @@ public abstract class TcpTransport extends AbstractLifecycleComponent implements
     private final TransportKeepAlive keepAlive;
     private final OutboundHandler outboundHandler;
     private final InboundHandler inboundHandler;
+    private final NativeOutboundHandler handshakerHandler;
     private final ResponseHandlers responseHandlers = new ResponseHandlers();
     private final RequestHandlers requestHandlers = new RequestHandlers();
 
@@ -186,11 +190,20 @@ public TcpTransport(
         }
         BigArrays bigArrays = new BigArrays(pageCacheRecycler, circuitBreakerService, CircuitBreaker.IN_FLIGHT_REQUESTS);
 
-        this.outboundHandler = new OutboundHandler(nodeName, version, features, statsTracker, threadPool, bigArrays);
+        this.outboundHandler = new OutboundHandler(statsTracker, threadPool);
+        this.handshakerHandler = new NativeOutboundHandler(
+            nodeName,
+            version,
+            features,
+            statsTracker,
+            threadPool,
+            bigArrays,
+            outboundHandler
+        );
         this.handshaker = new TransportHandshaker(
             version,
             threadPool,
-            (node, channel, requestId, v) -> outboundHandler.sendRequest(
+            (node, channel, requestId, v) -> handshakerHandler.sendRequest(
                 node,
                 channel,
                 requestId,
@@ -204,7 +217,12 @@ public TcpTransport(
         );
         this.keepAlive = new TransportKeepAlive(threadPool, this.outboundHandler::sendBytes);
         this.inboundHandler = new InboundHandler(
+            nodeName,
+            version,
+            features,
+            statsTracker,
             threadPool,
+            bigArrays,
             outboundHandler,
             namedWriteableRegistry,
             handshaker,
@@ -236,7 +254,7 @@ protected void doStart() {}
 
     @Override
     public synchronized void setMessageListener(TransportMessageListener listener) {
-        outboundHandler.setMessageListener(listener);
+        handshakerHandler.setMessageListener(listener);
         inboundHandler.setMessageListener(listener);
     }
 
@@ -317,7 +335,7 @@ public void sendRequest(long requestId, String action, TransportRequest request,
                 throw new NodeNotConnectedException(node, "connection already closed");
             }
             TcpChannel channel = channel(options.type());
-            outboundHandler.sendRequest(node, channel, requestId, action, request, options, getVersion(), compress, false);
+            handshakerHandler.sendRequest(node, channel, requestId, action, request, options, getVersion(), compress, false);
         }
     }
 
@@ -765,7 +783,7 @@ protected void serverAcceptedChannel(TcpChannel channel) {
      * @param channel the channel the message is from
      * @param message the message
      */
-    public void inboundMessage(TcpChannel channel, InboundMessage message) {
+    public void inboundMessage(TcpChannel channel, ProtocolInboundMessage message) {
         try {
             inboundHandler.inboundMessage(channel, message);
         } catch (Exception e) {
@@ -966,7 +984,10 @@ public static Set<ProfileSettings> getProfileSettings(Settings settings) {
 
     /**
      * Representation of a transport profile settings for a {@code transport.profiles.$profilename.*}
+     *
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     public static final class ProfileSettings {
         public final String profileName;
         public final boolean tcpNoDelay;
diff --git a/server/src/main/java/org/opensearch/transport/TcpTransportChannel.java b/server/src/main/java/org/opensearch/transport/TcpTransportChannel.java
index 81de0af07ea7c..750fd50a4c44c 100644
--- a/server/src/main/java/org/opensearch/transport/TcpTransportChannel.java
+++ b/server/src/main/java/org/opensearch/transport/TcpTransportChannel.java
@@ -50,7 +50,7 @@
 public final class TcpTransportChannel extends BaseTcpTransportChannel {
 
     private final AtomicBoolean released = new AtomicBoolean();
-    private final OutboundHandler outboundHandler;
+    private final ProtocolOutboundHandler outboundHandler;
     private final String action;
     private final long requestId;
     private final Version version;
@@ -60,7 +60,7 @@ public final class TcpTransportChannel extends BaseTcpTransportChannel {
     private final Releasable breakerRelease;
 
     TcpTransportChannel(
-        OutboundHandler outboundHandler,
+        ProtocolOutboundHandler outboundHandler,
         TcpChannel channel,
         String action,
         long requestId,
diff --git a/server/src/main/java/org/opensearch/transport/Transport.java b/server/src/main/java/org/opensearch/transport/Transport.java
index 8abedff37db14..b89393615c95f 100644
--- a/server/src/main/java/org/opensearch/transport/Transport.java
+++ b/server/src/main/java/org/opensearch/transport/Transport.java
@@ -58,8 +58,9 @@
 /**
  * OpenSearch Transport Interface
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface Transport extends LifecycleComponent {
 
     /**
@@ -166,7 +167,10 @@ default Object getCacheKey() {
     /**
      * This class represents a response context that encapsulates the actual response handler, the action and the connection it was
      * executed on.
+     *
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     final class ResponseContext<T extends TransportResponse> {
 
         private final TransportResponseHandler<T> handler;
@@ -196,7 +200,10 @@ public String action() {
 
     /**
      * This class is a registry that allows
+     *
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     final class ResponseHandlers {
         private final ConcurrentMapLong<ResponseContext<? extends TransportResponse>> handlers = ConcurrentCollections
             .newConcurrentMapLongWithAggressiveConcurrency();
@@ -276,8 +283,9 @@ public TransportResponseHandler<? extends TransportResponse> onResponseReceived(
     /**
      * Request handler implementations
      *
-     * @opensearch.internal
+     * @opensearch.api
      */
+    @PublicApi(since = "1.0.0")
     final class RequestHandlers {
 
         private volatile Map<String, RequestHandlerRegistry<? extends TransportRequest>> requestHandlers = Collections.emptyMap();
diff --git a/server/src/main/java/org/opensearch/transport/TransportAdapterProvider.java b/server/src/main/java/org/opensearch/transport/TransportAdapterProvider.java
new file mode 100644
index 0000000000000..36dbd5a699b40
--- /dev/null
+++ b/server/src/main/java/org/opensearch/transport/TransportAdapterProvider.java
@@ -0,0 +1,40 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.transport;
+
+import org.opensearch.common.annotation.ExperimentalApi;
+import org.opensearch.common.settings.Settings;
+
+import java.util.Optional;
+
+/**
+ * Transport specific adapter providers which could be injected into the transport processing chain. The transport adapters
+ * are transport specific and do not have any common abstraction on top.
+ * @param <T> transport type
+ *
+ * @opensearch.experimental
+ */
+@ExperimentalApi
+public interface TransportAdapterProvider<T> {
+    /**
+     * The name of this transport adapter provider (and essentially are freestyle).
+     * @return the name of this transport adapter provider
+     */
+    String name();
+
+    /**
+     * Provides a new transport adapter of required transport adapter class and transport instance.
+     * @param <C> transport adapter class
+     * @param settings settings
+     * @param transport HTTP transport instance
+     * @param adapterClass required transport adapter class
+     * @return the non-empty {@link Optional} if the transport adapter could be created, empty one otherwise
+     */
+    <C> Optional<C> create(Settings settings, T transport, Class<C> adapterClass);
+}
diff --git a/server/src/main/java/org/opensearch/transport/TransportChannel.java b/server/src/main/java/org/opensearch/transport/TransportChannel.java
index f84ee5dc745c3..7b6715ff2c73d 100644
--- a/server/src/main/java/org/opensearch/transport/TransportChannel.java
+++ b/server/src/main/java/org/opensearch/transport/TransportChannel.java
@@ -36,6 +36,7 @@
 import org.apache.logging.log4j.Logger;
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.opensearch.Version;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.transport.TransportResponse;
 
 import java.io.IOException;
@@ -46,6 +47,7 @@
  *
  * @opensearch.internal
  */
+@PublicApi(since = "1.0.0")
 public interface TransportChannel {
 
     Logger logger = LogManager.getLogger(TransportChannel.class);
diff --git a/server/src/main/java/org/opensearch/transport/TransportLogger.java b/server/src/main/java/org/opensearch/transport/TransportLogger.java
index 997b3bb5ba18e..e780f643aafd7 100644
--- a/server/src/main/java/org/opensearch/transport/TransportLogger.java
+++ b/server/src/main/java/org/opensearch/transport/TransportLogger.java
@@ -40,6 +40,7 @@
 import org.opensearch.core.common.io.stream.InputStreamStreamInput;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.compress.CompressorRegistry;
+import org.opensearch.transport.nativeprotocol.NativeInboundMessage;
 
 import java.io.IOException;
 
@@ -64,7 +65,7 @@ static void logInboundMessage(TcpChannel channel, BytesReference message) {
         }
     }
 
-    static void logInboundMessage(TcpChannel channel, InboundMessage message) {
+    static void logInboundMessage(TcpChannel channel, NativeInboundMessage message) {
         if (logger.isTraceEnabled()) {
             try {
                 String logMessage = format(channel, message, "READ");
@@ -136,7 +137,7 @@ private static String format(TcpChannel channel, BytesReference message, String
         return sb.toString();
     }
 
-    private static String format(TcpChannel channel, InboundMessage message, String event) throws IOException {
+    private static String format(TcpChannel channel, NativeInboundMessage message, String event) throws IOException {
         final StringBuilder sb = new StringBuilder();
         sb.append(channel);
 
diff --git a/server/src/main/java/org/opensearch/transport/TransportMessageListener.java b/server/src/main/java/org/opensearch/transport/TransportMessageListener.java
index dfcd7acce3706..284c4646655c5 100644
--- a/server/src/main/java/org/opensearch/transport/TransportMessageListener.java
+++ b/server/src/main/java/org/opensearch/transport/TransportMessageListener.java
@@ -32,13 +32,15 @@
 package org.opensearch.transport;
 
 import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.transport.TransportResponse;
 
 /**
  * Listens for transport messages
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface TransportMessageListener {
 
     TransportMessageListener NOOP_LISTENER = new TransportMessageListener() {
diff --git a/server/src/main/java/org/opensearch/transport/TransportRequestHandler.java b/server/src/main/java/org/opensearch/transport/TransportRequestHandler.java
index 54ee1b68fc9aa..0419c0b82029b 100644
--- a/server/src/main/java/org/opensearch/transport/TransportRequestHandler.java
+++ b/server/src/main/java/org/opensearch/transport/TransportRequestHandler.java
@@ -32,13 +32,15 @@
 
 package org.opensearch.transport;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.tasks.Task;
 
 /**
  * Handles transport requests
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface TransportRequestHandler<T extends TransportRequest> {
 
     void messageReceived(T request, TransportChannel channel, Task task) throws Exception;
diff --git a/server/src/main/java/org/opensearch/transport/TransportResponseHandler.java b/server/src/main/java/org/opensearch/transport/TransportResponseHandler.java
index 8992af18edb48..748d2a4d867ec 100644
--- a/server/src/main/java/org/opensearch/transport/TransportResponseHandler.java
+++ b/server/src/main/java/org/opensearch/transport/TransportResponseHandler.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.transport;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.Writeable;
 import org.opensearch.core.transport.TransportResponse;
@@ -42,8 +43,9 @@
 /**
  * Handles transport responses
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public interface TransportResponseHandler<T extends TransportResponse> extends Writeable.Reader<T> {
 
     void handleResponse(T response);
diff --git a/server/src/main/java/org/opensearch/transport/TransportStats.java b/server/src/main/java/org/opensearch/transport/TransportStats.java
index e3c4773f4a472..01980ce529caa 100644
--- a/server/src/main/java/org/opensearch/transport/TransportStats.java
+++ b/server/src/main/java/org/opensearch/transport/TransportStats.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.transport;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
@@ -44,8 +45,9 @@
 /**
  * Stats for transport activity
  *
- * @opensearch.internal
+ * @opensearch.api
  */
+@PublicApi(since = "1.0.0")
 public class TransportStats implements Writeable, ToXContentFragment {
 
     private final long serverOpen;
diff --git a/server/src/main/java/org/opensearch/transport/TransportStatus.java b/server/src/main/java/org/opensearch/transport/TransportStatus.java
index dab572949e688..76377468535b9 100644
--- a/server/src/main/java/org/opensearch/transport/TransportStatus.java
+++ b/server/src/main/java/org/opensearch/transport/TransportStatus.java
@@ -76,11 +76,11 @@ public static byte setCompress(byte value) {
         return value;
     }
 
-    static boolean isHandshake(byte value) { // pkg private since it's only used internally
+    public static boolean isHandshake(byte value) {
         return (value & STATUS_HANDSHAKE) != 0;
     }
 
-    static byte setHandshake(byte value) { // pkg private since it's only used internally
+    public static byte setHandshake(byte value) {
         value |= STATUS_HANDSHAKE;
         return value;
     }
diff --git a/server/src/main/java/org/opensearch/transport/CompressibleBytesOutputStream.java b/server/src/main/java/org/opensearch/transport/nativeprotocol/CompressibleBytesOutputStream.java
similarity index 98%
rename from server/src/main/java/org/opensearch/transport/CompressibleBytesOutputStream.java
rename to server/src/main/java/org/opensearch/transport/nativeprotocol/CompressibleBytesOutputStream.java
index 57707d3b44477..92b682370bcd5 100644
--- a/server/src/main/java/org/opensearch/transport/CompressibleBytesOutputStream.java
+++ b/server/src/main/java/org/opensearch/transport/nativeprotocol/CompressibleBytesOutputStream.java
@@ -30,7 +30,7 @@
  * GitHub history for details.
  */
 
-package org.opensearch.transport;
+package org.opensearch.transport.nativeprotocol;
 
 import org.opensearch.common.io.Streams;
 import org.opensearch.common.util.io.IOUtils;
diff --git a/server/src/main/java/org/opensearch/transport/nativeprotocol/NativeInboundBytesHandler.java b/server/src/main/java/org/opensearch/transport/nativeprotocol/NativeInboundBytesHandler.java
new file mode 100644
index 0000000000000..97981aeb6736e
--- /dev/null
+++ b/server/src/main/java/org/opensearch/transport/nativeprotocol/NativeInboundBytesHandler.java
@@ -0,0 +1,166 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.transport.nativeprotocol;
+
+import org.opensearch.common.bytes.ReleasableBytesReference;
+import org.opensearch.common.lease.Releasable;
+import org.opensearch.common.lease.Releasables;
+import org.opensearch.core.common.bytes.CompositeBytesReference;
+import org.opensearch.transport.Header;
+import org.opensearch.transport.InboundAggregator;
+import org.opensearch.transport.InboundBytesHandler;
+import org.opensearch.transport.InboundDecoder;
+import org.opensearch.transport.ProtocolInboundMessage;
+import org.opensearch.transport.StatsTracker;
+import org.opensearch.transport.TcpChannel;
+
+import java.io.IOException;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.function.BiConsumer;
+
+/**
+ * Handler for inbound bytes for the native protocol.
+ */
+public class NativeInboundBytesHandler implements InboundBytesHandler {
+
+    private static final ThreadLocal<ArrayList<Object>> fragmentList = ThreadLocal.withInitial(ArrayList::new);
+    private static final NativeInboundMessage PING_MESSAGE = new NativeInboundMessage(null, true);
+
+    private final ArrayDeque<ReleasableBytesReference> pending;
+    private final InboundDecoder decoder;
+    private final InboundAggregator aggregator;
+    private final StatsTracker statsTracker;
+    private boolean isClosed = false;
+
+    public NativeInboundBytesHandler(
+        ArrayDeque<ReleasableBytesReference> pending,
+        InboundDecoder decoder,
+        InboundAggregator aggregator,
+        StatsTracker statsTracker
+    ) {
+        this.pending = pending;
+        this.decoder = decoder;
+        this.aggregator = aggregator;
+        this.statsTracker = statsTracker;
+    }
+
+    @Override
+    public void close() {
+        isClosed = true;
+    }
+
+    @Override
+    public boolean canHandleBytes(ReleasableBytesReference reference) {
+        return true;
+    }
+
+    @Override
+    public void doHandleBytes(
+        TcpChannel channel,
+        ReleasableBytesReference reference,
+        BiConsumer<TcpChannel, ProtocolInboundMessage> messageHandler
+    ) throws IOException {
+        final ArrayList<Object> fragments = fragmentList.get();
+        boolean continueHandling = true;
+
+        while (continueHandling && isClosed == false) {
+            boolean continueDecoding = true;
+            while (continueDecoding && pending.isEmpty() == false) {
+                try (ReleasableBytesReference toDecode = getPendingBytes()) {
+                    final int bytesDecoded = decoder.decode(toDecode, fragments::add);
+                    if (bytesDecoded != 0) {
+                        releasePendingBytes(bytesDecoded);
+                        if (fragments.isEmpty() == false && endOfMessage(fragments.get(fragments.size() - 1))) {
+                            continueDecoding = false;
+                        }
+                    } else {
+                        continueDecoding = false;
+                    }
+                }
+            }
+
+            if (fragments.isEmpty()) {
+                continueHandling = false;
+            } else {
+                try {
+                    forwardFragments(channel, fragments, messageHandler);
+                } finally {
+                    for (Object fragment : fragments) {
+                        if (fragment instanceof ReleasableBytesReference) {
+                            ((ReleasableBytesReference) fragment).close();
+                        }
+                    }
+                    fragments.clear();
+                }
+            }
+        }
+    }
+
+    private ReleasableBytesReference getPendingBytes() {
+        if (pending.size() == 1) {
+            return pending.peekFirst().retain();
+        } else {
+            final ReleasableBytesReference[] bytesReferences = new ReleasableBytesReference[pending.size()];
+            int index = 0;
+            for (ReleasableBytesReference pendingReference : pending) {
+                bytesReferences[index] = pendingReference.retain();
+                ++index;
+            }
+            final Releasable releasable = () -> Releasables.closeWhileHandlingException(bytesReferences);
+            return new ReleasableBytesReference(CompositeBytesReference.of(bytesReferences), releasable);
+        }
+    }
+
+    private void releasePendingBytes(int bytesConsumed) {
+        int bytesToRelease = bytesConsumed;
+        while (bytesToRelease != 0) {
+            try (ReleasableBytesReference reference = pending.pollFirst()) {
+                assert reference != null;
+                if (bytesToRelease < reference.length()) {
+                    pending.addFirst(reference.retainedSlice(bytesToRelease, reference.length() - bytesToRelease));
+                    bytesToRelease -= bytesToRelease;
+                } else {
+                    bytesToRelease -= reference.length();
+                }
+            }
+        }
+    }
+
+    private boolean endOfMessage(Object fragment) {
+        return fragment == InboundDecoder.PING || fragment == InboundDecoder.END_CONTENT || fragment instanceof Exception;
+    }
+
+    private void forwardFragments(
+        TcpChannel channel,
+        ArrayList<Object> fragments,
+        BiConsumer<TcpChannel, ProtocolInboundMessage> messageHandler
+    ) throws IOException {
+        for (Object fragment : fragments) {
+            if (fragment instanceof Header) {
+                assert aggregator.isAggregating() == false;
+                aggregator.headerReceived((Header) fragment);
+            } else if (fragment == InboundDecoder.PING) {
+                assert aggregator.isAggregating() == false;
+                messageHandler.accept(channel, PING_MESSAGE);
+            } else if (fragment == InboundDecoder.END_CONTENT) {
+                assert aggregator.isAggregating();
+                try (NativeInboundMessage aggregated = aggregator.finishAggregation()) {
+                    statsTracker.markMessageReceived();
+                    messageHandler.accept(channel, aggregated);
+                }
+            } else {
+                assert aggregator.isAggregating();
+                assert fragment instanceof ReleasableBytesReference;
+                aggregator.aggregate((ReleasableBytesReference) fragment);
+            }
+        }
+    }
+
+}
diff --git a/server/src/main/java/org/opensearch/transport/InboundMessage.java b/server/src/main/java/org/opensearch/transport/nativeprotocol/NativeInboundMessage.java
similarity index 81%
rename from server/src/main/java/org/opensearch/transport/InboundMessage.java
rename to server/src/main/java/org/opensearch/transport/nativeprotocol/NativeInboundMessage.java
index a1ed682ff7d7f..1143f129b6319 100644
--- a/server/src/main/java/org/opensearch/transport/InboundMessage.java
+++ b/server/src/main/java/org/opensearch/transport/nativeprotocol/NativeInboundMessage.java
@@ -30,22 +30,31 @@
  * GitHub history for details.
  */
 
-package org.opensearch.transport;
+package org.opensearch.transport.nativeprotocol;
 
+import org.opensearch.common.annotation.PublicApi;
 import org.opensearch.common.bytes.ReleasableBytesReference;
 import org.opensearch.common.lease.Releasable;
 import org.opensearch.common.lease.Releasables;
 import org.opensearch.common.util.io.IOUtils;
 import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.transport.Header;
+import org.opensearch.transport.ProtocolInboundMessage;
 
 import java.io.IOException;
 
 /**
  * Inbound data as a message
  *
- * @opensearch.internal
+ * @opensearch.api
  */
-public class InboundMessage implements Releasable {
+@PublicApi(since = "2.14.0")
+public class NativeInboundMessage implements Releasable, ProtocolInboundMessage {
+
+    /**
+     * The protocol used to encode this message
+     */
+    public static String NATIVE_PROTOCOL = "native";
 
     private final Header header;
     private final ReleasableBytesReference content;
@@ -54,7 +63,7 @@ public class InboundMessage implements Releasable {
     private Releasable breakerRelease;
     private StreamInput streamInput;
 
-    public InboundMessage(Header header, ReleasableBytesReference content, Releasable breakerRelease) {
+    public NativeInboundMessage(Header header, ReleasableBytesReference content, Releasable breakerRelease) {
         this.header = header;
         this.content = content;
         this.breakerRelease = breakerRelease;
@@ -62,7 +71,7 @@ public InboundMessage(Header header, ReleasableBytesReference content, Releasabl
         this.isPing = false;
     }
 
-    public InboundMessage(Header header, Exception exception) {
+    public NativeInboundMessage(Header header, Exception exception) {
         this.header = header;
         this.content = null;
         this.breakerRelease = null;
@@ -70,7 +79,7 @@ public InboundMessage(Header header, Exception exception) {
         this.isPing = false;
     }
 
-    public InboundMessage(Header header, boolean isPing) {
+    public NativeInboundMessage(Header header, boolean isPing) {
         this.header = header;
         this.content = null;
         this.breakerRelease = null;
@@ -78,6 +87,11 @@ public InboundMessage(Header header, boolean isPing) {
         this.isPing = isPing;
     }
 
+    @Override
+    public String getProtocol() {
+        return NATIVE_PROTOCOL;
+    }
+
     public Header getHeader() {
         return header;
     }
@@ -131,4 +145,5 @@ public void close() {
     public String toString() {
         return "InboundMessage{" + header + "}";
     }
+
 }
diff --git a/server/src/main/java/org/opensearch/transport/nativeprotocol/NativeOutboundHandler.java b/server/src/main/java/org/opensearch/transport/nativeprotocol/NativeOutboundHandler.java
new file mode 100644
index 0000000000000..66ed0d8e3eb2b
--- /dev/null
+++ b/server/src/main/java/org/opensearch/transport/nativeprotocol/NativeOutboundHandler.java
@@ -0,0 +1,224 @@
+/*
+* SPDX-License-Identifier: Apache-2.0
+*
+* The OpenSearch Contributors require contributions made to
+* this file be licensed under the Apache-2.0 license or a
+* compatible open source license.
+*/
+
+/*
+* Licensed to Elasticsearch under one or more contributor
+* license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright
+* ownership. Elasticsearch licenses this file to you under
+* the Apache License, Version 2.0 (the "License"); you may
+* not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+*    http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing,
+* software distributed under the License is distributed on an
+* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+* KIND, either express or implied.  See the License for the
+* specific language governing permissions and limitations
+* under the License.
+*/
+
+/*
+* Modifications Copyright OpenSearch Contributors. See
+* GitHub history for details.
+*/
+
+package org.opensearch.transport.nativeprotocol;
+
+import org.opensearch.Version;
+import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.common.CheckedSupplier;
+import org.opensearch.common.io.stream.ReleasableBytesStreamOutput;
+import org.opensearch.common.lease.Releasable;
+import org.opensearch.common.util.BigArrays;
+import org.opensearch.common.util.io.IOUtils;
+import org.opensearch.core.action.ActionListener;
+import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.core.common.transport.TransportAddress;
+import org.opensearch.core.transport.TransportResponse;
+import org.opensearch.threadpool.ThreadPool;
+import org.opensearch.transport.OutboundHandler;
+import org.opensearch.transport.ProtocolOutboundHandler;
+import org.opensearch.transport.RemoteTransportException;
+import org.opensearch.transport.StatsTracker;
+import org.opensearch.transport.TcpChannel;
+import org.opensearch.transport.TransportException;
+import org.opensearch.transport.TransportMessageListener;
+import org.opensearch.transport.TransportRequest;
+import org.opensearch.transport.TransportRequestOptions;
+
+import java.io.IOException;
+import java.util.Set;
+
+/**
+ * Outbound data handler
+ *
+ * @opensearch.internal
+ */
+public final class NativeOutboundHandler extends ProtocolOutboundHandler {
+    private final String nodeName;
+    private final Version version;
+    private final String[] features;
+    private final StatsTracker statsTracker;
+    private final ThreadPool threadPool;
+    private final BigArrays bigArrays;
+    private volatile TransportMessageListener messageListener = TransportMessageListener.NOOP_LISTENER;
+    private final OutboundHandler handler;
+
+    public NativeOutboundHandler(
+        String nodeName,
+        Version version,
+        String[] features,
+        StatsTracker statsTracker,
+        ThreadPool threadPool,
+        BigArrays bigArrays,
+        OutboundHandler handler
+    ) {
+        this.nodeName = nodeName;
+        this.version = version;
+        this.features = features;
+        this.statsTracker = statsTracker;
+        this.threadPool = threadPool;
+        this.bigArrays = bigArrays;
+        this.handler = handler;
+    }
+
+    /**
+     * Sends the request to the given channel. This method should be used to send {@link TransportRequest}
+     * objects back to the caller.
+     */
+    @Override
+    public void sendRequest(
+        final DiscoveryNode node,
+        final TcpChannel channel,
+        final long requestId,
+        final String action,
+        final TransportRequest request,
+        final TransportRequestOptions options,
+        final Version channelVersion,
+        final boolean compressRequest,
+        final boolean isHandshake
+    ) throws IOException, TransportException {
+        Version version = Version.min(this.version, channelVersion);
+        NativeOutboundMessage.Request message = new NativeOutboundMessage.Request(
+            threadPool.getThreadContext(),
+            features,
+            request,
+            version,
+            action,
+            requestId,
+            isHandshake,
+            compressRequest
+        );
+        ActionListener<Void> listener = ActionListener.wrap(() -> messageListener.onRequestSent(node, requestId, action, request, options));
+        sendMessage(channel, message, listener);
+    }
+
+    /**
+     * Sends the response to the given channel. This method should be used to send {@link TransportResponse}
+     * objects back to the caller.
+     *
+     * @see #sendErrorResponse(Version, Set, TcpChannel, long, String, Exception) for sending error responses
+     */
+    @Override
+    public void sendResponse(
+        final Version nodeVersion,
+        final Set<String> features,
+        final TcpChannel channel,
+        final long requestId,
+        final String action,
+        final TransportResponse response,
+        final boolean compress,
+        final boolean isHandshake
+    ) throws IOException {
+        Version version = Version.min(this.version, nodeVersion);
+        NativeOutboundMessage.Response message = new NativeOutboundMessage.Response(
+            threadPool.getThreadContext(),
+            features,
+            response,
+            version,
+            requestId,
+            isHandshake,
+            compress
+        );
+        ActionListener<Void> listener = ActionListener.wrap(() -> messageListener.onResponseSent(requestId, action, response));
+        sendMessage(channel, message, listener);
+    }
+
+    /**
+     * Sends back an error response to the caller via the given channel
+     */
+    @Override
+    public void sendErrorResponse(
+        final Version nodeVersion,
+        final Set<String> features,
+        final TcpChannel channel,
+        final long requestId,
+        final String action,
+        final Exception error
+    ) throws IOException {
+        Version version = Version.min(this.version, nodeVersion);
+        TransportAddress address = new TransportAddress(channel.getLocalAddress());
+        RemoteTransportException tx = new RemoteTransportException(nodeName, address, action, error);
+        NativeOutboundMessage.Response message = new NativeOutboundMessage.Response(
+            threadPool.getThreadContext(),
+            features,
+            tx,
+            version,
+            requestId,
+            false,
+            false
+        );
+        ActionListener<Void> listener = ActionListener.wrap(() -> messageListener.onResponseSent(requestId, action, error));
+        sendMessage(channel, message, listener);
+    }
+
+    private void sendMessage(TcpChannel channel, NativeOutboundMessage networkMessage, ActionListener<Void> listener) throws IOException {
+        MessageSerializer serializer = new MessageSerializer(networkMessage, bigArrays);
+        OutboundHandler.SendContext sendContext = new OutboundHandler.SendContext(statsTracker, channel, serializer, listener, serializer);
+        handler.sendBytes(channel, sendContext);
+    }
+
+    public void setMessageListener(TransportMessageListener listener) {
+        if (messageListener == TransportMessageListener.NOOP_LISTENER) {
+            messageListener = listener;
+        } else {
+            throw new IllegalStateException("Cannot set message listener twice");
+        }
+    }
+
+    /**
+     * Internal message serializer
+     *
+     * @opensearch.internal
+     */
+    private static class MessageSerializer implements CheckedSupplier<BytesReference, IOException>, Releasable {
+
+        private final NativeOutboundMessage message;
+        private final BigArrays bigArrays;
+        private volatile ReleasableBytesStreamOutput bytesStreamOutput;
+
+        private MessageSerializer(NativeOutboundMessage message, BigArrays bigArrays) {
+            this.message = message;
+            this.bigArrays = bigArrays;
+        }
+
+        @Override
+        public BytesReference get() throws IOException {
+            bytesStreamOutput = new ReleasableBytesStreamOutput(bigArrays);
+            return message.serialize(bytesStreamOutput);
+        }
+
+        @Override
+        public void close() {
+            IOUtils.closeWhileHandlingException(bytesStreamOutput);
+        }
+    }
+}
diff --git a/server/src/main/java/org/opensearch/transport/OutboundMessage.java b/server/src/main/java/org/opensearch/transport/nativeprotocol/NativeOutboundMessage.java
similarity index 91%
rename from server/src/main/java/org/opensearch/transport/OutboundMessage.java
rename to server/src/main/java/org/opensearch/transport/nativeprotocol/NativeOutboundMessage.java
index 3dafc6fb5eea9..d7590fb9e03ab 100644
--- a/server/src/main/java/org/opensearch/transport/OutboundMessage.java
+++ b/server/src/main/java/org/opensearch/transport/nativeprotocol/NativeOutboundMessage.java
@@ -29,7 +29,7 @@
  * GitHub history for details.
  */
 
-package org.opensearch.transport;
+package org.opensearch.transport.nativeprotocol;
 
 import org.opensearch.Version;
 import org.opensearch.common.io.stream.BytesStreamOutput;
@@ -39,6 +39,10 @@
 import org.opensearch.core.common.bytes.CompositeBytesReference;
 import org.opensearch.core.common.io.stream.StreamOutput;
 import org.opensearch.core.common.io.stream.Writeable;
+import org.opensearch.transport.BytesTransportRequest;
+import org.opensearch.transport.RemoteTransportException;
+import org.opensearch.transport.TcpHeader;
+import org.opensearch.transport.TransportStatus;
 
 import java.io.IOException;
 import java.util.Set;
@@ -48,11 +52,11 @@
  *
  * @opensearch.internal
  */
-abstract class OutboundMessage extends NetworkMessage {
+abstract class NativeOutboundMessage extends NetworkMessage {
 
     private final Writeable message;
 
-    OutboundMessage(ThreadContext threadContext, Version version, byte status, long requestId, Writeable message) {
+    NativeOutboundMessage(ThreadContext threadContext, Version version, byte status, long requestId, Writeable message) {
         super(threadContext, version, status, requestId);
         this.message = message;
     }
@@ -93,7 +97,7 @@ protected BytesReference writeMessage(CompressibleBytesOutputStream stream) thro
         if (message instanceof BytesTransportRequest) {
             BytesTransportRequest bRequest = (BytesTransportRequest) message;
             bRequest.writeThin(stream);
-            zeroCopyBuffer = bRequest.bytes;
+            zeroCopyBuffer = bRequest.bytes();
         } else if (message instanceof RemoteTransportException) {
             stream.writeException((RemoteTransportException) message);
             zeroCopyBuffer = BytesArray.EMPTY;
@@ -119,7 +123,7 @@ protected BytesReference writeMessage(CompressibleBytesOutputStream stream) thro
      *
      * @opensearch.internal
      */
-    static class Request extends OutboundMessage {
+    static class Request extends NativeOutboundMessage {
 
         private final String[] features;
         private final String action;
@@ -149,7 +153,7 @@ protected void writeVariableHeader(StreamOutput stream) throws IOException {
         private static byte setStatus(boolean compress, boolean isHandshake, Writeable message) {
             byte status = 0;
             status = TransportStatus.setRequest(status);
-            if (compress && OutboundMessage.canCompress(message)) {
+            if (compress && NativeOutboundMessage.canCompress(message)) {
                 status = TransportStatus.setCompress(status);
             }
             if (isHandshake) {
@@ -165,7 +169,7 @@ private static byte setStatus(boolean compress, boolean isHandshake, Writeable m
      *
      * @opensearch.internal
      */
-    static class Response extends OutboundMessage {
+    static class Response extends NativeOutboundMessage {
 
         private final Set<String> features;
 
diff --git a/server/src/main/java/org/opensearch/transport/NetworkMessage.java b/server/src/main/java/org/opensearch/transport/nativeprotocol/NetworkMessage.java
similarity index 96%
rename from server/src/main/java/org/opensearch/transport/NetworkMessage.java
rename to server/src/main/java/org/opensearch/transport/nativeprotocol/NetworkMessage.java
index f02d664b65929..c197539d2e009 100644
--- a/server/src/main/java/org/opensearch/transport/NetworkMessage.java
+++ b/server/src/main/java/org/opensearch/transport/nativeprotocol/NetworkMessage.java
@@ -29,11 +29,12 @@
  * GitHub history for details.
  */
 
-package org.opensearch.transport;
+package org.opensearch.transport.nativeprotocol;
 
 import org.opensearch.Version;
 import org.opensearch.common.util.concurrent.ThreadContext;
 import org.opensearch.core.common.io.stream.Writeable;
+import org.opensearch.transport.TransportStatus;
 
 /**
  * Represents a transport message sent over the network. Subclasses implement serialization and
diff --git a/server/src/main/java/org/opensearch/transport/nativeprotocol/package-info.java b/server/src/main/java/org/opensearch/transport/nativeprotocol/package-info.java
new file mode 100644
index 0000000000000..84f6d7d0ec5c2
--- /dev/null
+++ b/server/src/main/java/org/opensearch/transport/nativeprotocol/package-info.java
@@ -0,0 +1,10 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/** Native transport protocol package. */
+package org.opensearch.transport.nativeprotocol;
diff --git a/server/src/main/resources/org/opensearch/bootstrap/security.policy b/server/src/main/resources/org/opensearch/bootstrap/security.policy
index e1226345ef961..55e8db0d9c6a3 100644
--- a/server/src/main/resources/org/opensearch/bootstrap/security.policy
+++ b/server/src/main/resources/org/opensearch/bootstrap/security.policy
@@ -85,6 +85,11 @@ grant codeBase "${codebase.zstd-jni}" {
   permission java.lang.RuntimePermission "loadLibrary.*";
 };
 
+// repository-azure plugin and server side streaming
+grant codeBase "${codebase.reactor-core}" {
+  permission java.net.SocketPermission "*", "connect,resolve";
+};
+
 //// Everything else:
 
 grant {
diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java b/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java
index 1b8b6243aa805..14bfc85abac16 100644
--- a/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java
+++ b/server/src/test/java/org/opensearch/action/admin/cluster/node/stats/NodeStatsTests.java
@@ -42,6 +42,12 @@
 import org.opensearch.cluster.routing.WeightedRoutingStats;
 import org.opensearch.cluster.service.ClusterManagerThrottlingStats;
 import org.opensearch.cluster.service.ClusterStateStats;
+import org.opensearch.common.cache.CacheType;
+import org.opensearch.common.cache.service.NodeCacheStats;
+import org.opensearch.common.cache.stats.CacheStats;
+import org.opensearch.common.cache.stats.DefaultCacheStatsHolder;
+import org.opensearch.common.cache.stats.DefaultCacheStatsHolderTests;
+import org.opensearch.common.cache.stats.ImmutableCacheStatsHolder;
 import org.opensearch.common.io.stream.BytesStreamOutput;
 import org.opensearch.common.metrics.OperationStats;
 import org.opensearch.common.settings.ClusterSettings;
@@ -89,6 +95,7 @@
 import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
+import java.util.TreeMap;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 import java.util.stream.StreamSupport;
@@ -577,6 +584,13 @@ public void testSerialization() throws IOException {
                         deserializedAdmissionControllerStats.getRejectionCount().get(AdmissionControlActionType.INDEXING.getType())
                     );
                 }
+                NodeCacheStats nodeCacheStats = nodeStats.getNodeCacheStats();
+                NodeCacheStats deserializedNodeCacheStats = deserializedNodeStats.getNodeCacheStats();
+                if (nodeCacheStats == null) {
+                    assertNull(deserializedNodeCacheStats);
+                } else {
+                    assertEquals(nodeCacheStats, deserializedNodeCacheStats);
+                }
             }
         }
     }
@@ -928,6 +942,39 @@ public void apply(String action, AdmissionControlActionType admissionControlActi
 
         NodeIndicesStats indicesStats = getNodeIndicesStats(remoteStoreStats);
 
+        NodeCacheStats nodeCacheStats = null;
+        if (frequently()) {
+            int numIndices = randomIntBetween(1, 10);
+            int numShardsPerIndex = randomIntBetween(1, 50);
+
+            List<String> dimensionNames = List.of("index", "shard", "tier");
+            DefaultCacheStatsHolder statsHolder = new DefaultCacheStatsHolder(dimensionNames, "dummyStoreName");
+            for (int indexNum = 0; indexNum < numIndices; indexNum++) {
+                String indexName = "index" + indexNum;
+                for (int shardNum = 0; shardNum < numShardsPerIndex; shardNum++) {
+                    String shardName = "[" + indexName + "][" + shardNum + "]";
+                    for (String tierName : new String[] { "dummy_tier_1", "dummy_tier_2" }) {
+                        List<String> dimensionValues = List.of(indexName, shardName, tierName);
+                        CacheStats toIncrement = new CacheStats(randomInt(20), randomInt(20), randomInt(20), randomInt(20), randomInt(20));
+                        DefaultCacheStatsHolderTests.populateStatsHolderFromStatsValueMap(
+                            statsHolder,
+                            Map.of(dimensionValues, toIncrement)
+                        );
+                    }
+                }
+            }
+            CommonStatsFlags flags = new CommonStatsFlags();
+            for (CacheType cacheType : CacheType.values()) {
+                if (frequently()) {
+                    flags.includeCacheType(cacheType);
+                }
+            }
+            ImmutableCacheStatsHolder cacheStats = statsHolder.getImmutableCacheStatsHolder(dimensionNames.toArray(new String[0]));
+            TreeMap<CacheType, ImmutableCacheStatsHolder> cacheStatsMap = new TreeMap<>();
+            cacheStatsMap.put(CacheType.INDICES_REQUEST_CACHE, cacheStats);
+            nodeCacheStats = new NodeCacheStats(cacheStatsMap, flags);
+        }
+
         // TODO: Only remote_store based aspects of NodeIndicesStats are being tested here.
         // It is possible to test other metrics in NodeIndicesStats as well since it extends Writeable now
         return new NodeStats(
@@ -958,7 +1005,8 @@ public void apply(String action, AdmissionControlActionType admissionControlActi
             null,
             segmentReplicationRejectionStats,
             null,
-            admissionControlStats
+            admissionControlStats,
+            nodeCacheStats
         );
     }
 
diff --git a/server/src/test/java/org/opensearch/action/admin/indices/close/TransportVerifyShardBeforeCloseActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/close/TransportVerifyShardBeforeCloseActionTests.java
index ef26bc225b0c7..5ca5f53f180be 100644
--- a/server/src/test/java/org/opensearch/action/admin/indices/close/TransportVerifyShardBeforeCloseActionTests.java
+++ b/server/src/test/java/org/opensearch/action/admin/indices/close/TransportVerifyShardBeforeCloseActionTests.java
@@ -82,6 +82,7 @@
 import org.mockito.ArgumentCaptor;
 
 import static org.opensearch.action.support.replication.ClusterStateCreationUtils.state;
+import static org.opensearch.index.remote.RemoteStoreTestsHelper.createIndexSettings;
 import static org.opensearch.test.ClusterServiceUtils.createClusterService;
 import static org.opensearch.test.ClusterServiceUtils.setState;
 import static org.hamcrest.Matchers.arrayWithSize;
@@ -332,15 +333,15 @@ public void testUnavailableShardsMarkedAsStale() throws Exception {
     public void testGetReplicationModeWithRemoteTranslog() {
         TransportVerifyShardBeforeCloseAction action = createAction();
         final IndexShard indexShard = mock(IndexShard.class);
-        when(indexShard.isRemoteTranslogEnabled()).thenReturn(true);
+        when(indexShard.indexSettings()).thenReturn(createIndexSettings(true));
         assertEquals(ReplicationMode.NO_REPLICATION, action.getReplicationMode(indexShard));
     }
 
     public void testGetReplicationModeWithLocalTranslog() {
         TransportVerifyShardBeforeCloseAction action = createAction();
         final IndexShard indexShard = mock(IndexShard.class);
-        when(indexShard.isRemoteTranslogEnabled()).thenReturn(true);
-        assertEquals(ReplicationMode.NO_REPLICATION, action.getReplicationMode(indexShard));
+        when(indexShard.indexSettings()).thenReturn(createIndexSettings(false));
+        assertEquals(ReplicationMode.FULL_REPLICATION, action.getReplicationMode(indexShard));
     }
 
     private TransportVerifyShardBeforeCloseAction createAction() {
diff --git a/server/src/test/java/org/opensearch/action/admin/indices/flush/TransportShardFlushActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/flush/TransportShardFlushActionTests.java
index 09215088bd04b..c9d3a6c4c7605 100644
--- a/server/src/test/java/org/opensearch/action/admin/indices/flush/TransportShardFlushActionTests.java
+++ b/server/src/test/java/org/opensearch/action/admin/indices/flush/TransportShardFlushActionTests.java
@@ -20,6 +20,7 @@
 import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.transport.TransportService;
 
+import static org.opensearch.index.remote.RemoteStoreTestsHelper.createIndexSettings;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -28,14 +29,14 @@ public class TransportShardFlushActionTests extends OpenSearchTestCase {
     public void testGetReplicationModeWithRemoteTranslog() {
         TransportShardFlushAction action = createAction();
         final IndexShard indexShard = mock(IndexShard.class);
-        when(indexShard.isRemoteTranslogEnabled()).thenReturn(true);
+        when(indexShard.indexSettings()).thenReturn(createIndexSettings(true));
         assertEquals(ReplicationMode.NO_REPLICATION, action.getReplicationMode(indexShard));
     }
 
     public void testGetReplicationModeWithLocalTranslog() {
         TransportShardFlushAction action = createAction();
         final IndexShard indexShard = mock(IndexShard.class);
-        when(indexShard.isRemoteTranslogEnabled()).thenReturn(false);
+        when(indexShard.indexSettings()).thenReturn(createIndexSettings(false));
         assertEquals(ReplicationMode.FULL_REPLICATION, action.getReplicationMode(indexShard));
     }
 
diff --git a/server/src/test/java/org/opensearch/action/admin/indices/forcemerge/RestForceMergeActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/forcemerge/RestForceMergeActionTests.java
index b09e592922ed9..01d72b78a679e 100644
--- a/server/src/test/java/org/opensearch/action/admin/indices/forcemerge/RestForceMergeActionTests.java
+++ b/server/src/test/java/org/opensearch/action/admin/indices/forcemerge/RestForceMergeActionTests.java
@@ -32,14 +32,9 @@
 
 package org.opensearch.action.admin.indices.forcemerge;
 
-import org.opensearch.client.node.NodeClient;
-import org.opensearch.common.xcontent.json.JsonXContent;
-import org.opensearch.core.common.bytes.BytesArray;
-import org.opensearch.core.xcontent.MediaTypeRegistry;
 import org.opensearch.core.xcontent.NamedXContentRegistry;
 import org.opensearch.rest.RestRequest;
 import org.opensearch.rest.action.admin.indices.RestForceMergeAction;
-import org.opensearch.test.rest.FakeRestChannel;
 import org.opensearch.test.rest.FakeRestRequest;
 import org.opensearch.test.rest.RestActionTestCase;
 import org.junit.Before;
@@ -47,9 +42,6 @@
 import java.util.HashMap;
 import java.util.Map;
 
-import static org.hamcrest.Matchers.equalTo;
-import static org.mockito.Mockito.mock;
-
 public class RestForceMergeActionTests extends RestActionTestCase {
 
     @Before
@@ -57,20 +49,6 @@ public void setUpAction() {
         controller().registerHandler(new RestForceMergeAction());
     }
 
-    public void testBodyRejection() throws Exception {
-        final RestForceMergeAction handler = new RestForceMergeAction();
-        String json = JsonXContent.contentBuilder().startObject().field("max_num_segments", 1).endObject().toString();
-        final FakeRestRequest request = new FakeRestRequest.Builder(NamedXContentRegistry.EMPTY).withContent(
-            new BytesArray(json),
-            MediaTypeRegistry.JSON
-        ).withPath("/_forcemerge").build();
-        IllegalArgumentException e = expectThrows(
-            IllegalArgumentException.class,
-            () -> handler.handleRequest(request, new FakeRestChannel(request, randomBoolean(), 1), mock(NodeClient.class))
-        );
-        assertThat(e.getMessage(), equalTo("request [GET /_forcemerge] does not support having a body"));
-    }
-
     public void testDeprecationMessage() {
         final Map<String, String> params = new HashMap<>();
         params.put("only_expunge_deletes", Boolean.TRUE.toString());
diff --git a/server/src/test/java/org/opensearch/action/admin/indices/readonly/TransportVerifyShardIndexBlockActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/readonly/TransportVerifyShardIndexBlockActionTests.java
index 8c4a6c023f9a5..90498d6d35700 100644
--- a/server/src/test/java/org/opensearch/action/admin/indices/readonly/TransportVerifyShardIndexBlockActionTests.java
+++ b/server/src/test/java/org/opensearch/action/admin/indices/readonly/TransportVerifyShardIndexBlockActionTests.java
@@ -20,6 +20,7 @@
 import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.transport.TransportService;
 
+import static org.opensearch.index.remote.RemoteStoreTestsHelper.createIndexSettings;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -28,14 +29,14 @@ public class TransportVerifyShardIndexBlockActionTests extends OpenSearchTestCas
     public void testGetReplicationModeWithRemoteTranslog() {
         TransportVerifyShardIndexBlockAction action = createAction();
         final IndexShard indexShard = mock(IndexShard.class);
-        when(indexShard.isRemoteTranslogEnabled()).thenReturn(true);
+        when(indexShard.indexSettings()).thenReturn(createIndexSettings(true));
         assertEquals(ReplicationMode.NO_REPLICATION, action.getReplicationMode(indexShard));
     }
 
     public void testGetReplicationModeWithLocalTranslog() {
         TransportVerifyShardIndexBlockAction action = createAction();
         final IndexShard indexShard = mock(IndexShard.class);
-        when(indexShard.isRemoteTranslogEnabled()).thenReturn(false);
+        when(indexShard.indexSettings()).thenReturn(createIndexSettings(false));
         assertEquals(ReplicationMode.FULL_REPLICATION, action.getReplicationMode(indexShard));
     }
 
diff --git a/server/src/test/java/org/opensearch/action/admin/indices/refresh/TransportShardRefreshActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/refresh/TransportShardRefreshActionTests.java
index b2eee904bad38..bc0b7e5cf14b2 100644
--- a/server/src/test/java/org/opensearch/action/admin/indices/refresh/TransportShardRefreshActionTests.java
+++ b/server/src/test/java/org/opensearch/action/admin/indices/refresh/TransportShardRefreshActionTests.java
@@ -20,6 +20,7 @@
 import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.transport.TransportService;
 
+import static org.opensearch.index.remote.RemoteStoreTestsHelper.createIndexSettings;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
@@ -28,14 +29,14 @@ public class TransportShardRefreshActionTests extends OpenSearchTestCase {
     public void testGetReplicationModeWithRemoteTranslog() {
         TransportShardRefreshAction action = createAction();
         final IndexShard indexShard = mock(IndexShard.class);
-        when(indexShard.isRemoteTranslogEnabled()).thenReturn(true);
+        when(indexShard.indexSettings()).thenReturn(createIndexSettings(true));
         assertEquals(ReplicationMode.NO_REPLICATION, action.getReplicationMode(indexShard));
     }
 
     public void testGetReplicationModeWithLocalTranslog() {
         TransportShardRefreshAction action = createAction();
         final IndexShard indexShard = mock(IndexShard.class);
-        when(indexShard.isRemoteTranslogEnabled()).thenReturn(false);
+        when(indexShard.indexSettings()).thenReturn(createIndexSettings(false));
         assertEquals(ReplicationMode.FULL_REPLICATION, action.getReplicationMode(indexShard));
     }
 
diff --git a/server/src/test/java/org/opensearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java b/server/src/test/java/org/opensearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java
index da9a8b928a779..50ffd7322544a 100644
--- a/server/src/test/java/org/opensearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java
+++ b/server/src/test/java/org/opensearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java
@@ -81,6 +81,7 @@
 import org.opensearch.index.mapper.MetadataFieldMapper;
 import org.opensearch.index.mapper.RoutingFieldMapper;
 import org.opensearch.index.shard.IndexEventListener;
+import org.opensearch.indices.DefaultRemoteStoreSettings;
 import org.opensearch.indices.IndicesService;
 import org.opensearch.indices.InvalidIndexNameException;
 import org.opensearch.indices.ShardLimitValidator;
@@ -738,7 +739,8 @@ public void testRolloverClusterState() throws Exception {
                 null,
                 systemIndices,
                 false,
-                new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings())
+                new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings()),
+                DefaultRemoteStoreSettings.INSTANCE
             );
             MetadataIndexAliasesService indexAliasesService = new MetadataIndexAliasesService(
                 clusterService,
@@ -876,7 +878,8 @@ public void testRolloverClusterStateForDataStream() throws Exception {
                 null,
                 systemIndices,
                 false,
-                new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings())
+                new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings()),
+                DefaultRemoteStoreSettings.INSTANCE
             );
             MetadataIndexAliasesService indexAliasesService = new MetadataIndexAliasesService(
                 clusterService,
@@ -1054,7 +1057,8 @@ public void testRolloverClusterStateForDataStreamNoTemplate() throws Exception {
             null,
             new SystemIndices(emptyMap()),
             false,
-            new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings())
+            new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings()),
+            DefaultRemoteStoreSettings.INSTANCE
         );
         MetadataIndexAliasesService indexAliasesService = new MetadataIndexAliasesService(
             clusterService,
diff --git a/server/src/test/java/org/opensearch/action/admin/indices/shrink/TransportResizeActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/shrink/TransportResizeActionTests.java
index 848df5f8e4979..5bab2ceca0988 100644
--- a/server/src/test/java/org/opensearch/action/admin/indices/shrink/TransportResizeActionTests.java
+++ b/server/src/test/java/org/opensearch/action/admin/indices/shrink/TransportResizeActionTests.java
@@ -51,10 +51,13 @@
 import org.opensearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
 import org.opensearch.cluster.routing.allocation.decider.AllocationDeciders;
 import org.opensearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider;
+import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.core.common.unit.ByteSizeValue;
 import org.opensearch.index.shard.DocsStats;
 import org.opensearch.index.store.StoreStats;
+import org.opensearch.node.remotestore.RemoteStoreNodeService;
 import org.opensearch.snapshots.EmptySnapshotsInfoService;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.test.gateway.TestGatewayAllocator;
@@ -65,7 +68,12 @@
 import java.util.Set;
 
 import static java.util.Collections.emptyMap;
-import static org.hamcrest.Matchers.equalTo;
+import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED;
+import static org.opensearch.common.util.FeatureFlags.REMOTE_STORE_MIGRATION_EXPERIMENTAL;
+import static org.opensearch.node.remotestore.RemoteStoreNodeService.CompatibilityMode;
+import static org.opensearch.node.remotestore.RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING;
+import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING;
+import static org.hamcrest.CoreMatchers.equalTo;
 
 public class TransportResizeActionTests extends OpenSearchTestCase {
 
@@ -95,6 +103,19 @@ private ClusterState createClusterState(String name, int numShards, int numRepli
         return clusterState;
     }
 
+    private ClusterSettings createClusterSettings(
+        CompatibilityMode compatibilityMode,
+        RemoteStoreNodeService.Direction migrationDirection
+    ) {
+        ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        clusterSettings.applySettings(
+            (Settings.builder()
+                .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), compatibilityMode)
+                .put(MIGRATION_DIRECTION_SETTING.getKey(), migrationDirection)).build()
+        );
+        return clusterSettings;
+    }
+
     public void testErrorCondition() {
         ClusterState state = createClusterState(
             "source",
@@ -102,6 +123,7 @@ public void testErrorCondition() {
             randomIntBetween(0, 10),
             Settings.builder().put("index.blocks.write", true).build()
         );
+        ClusterSettings clusterSettings = createClusterSettings(CompatibilityMode.STRICT, RemoteStoreNodeService.Direction.NONE);
         assertTrue(
             expectThrows(
                 IllegalStateException.class,
@@ -110,6 +132,7 @@ public void testErrorCondition() {
                     state,
                     (i) -> new DocsStats(Integer.MAX_VALUE, between(1, 1000), between(1, 100)),
                     new StoreStats(between(1, 10000), between(1, 10000)),
+                    clusterSettings,
                     "source",
                     "target"
                 )
@@ -125,6 +148,7 @@ public void testErrorCondition() {
                 clusterState,
                 (i) -> i == 2 || i == 3 ? new DocsStats(Integer.MAX_VALUE / 2, between(1, 1000), between(1, 10000)) : null,
                 new StoreStats(between(1, 10000), between(1, 10000)),
+                clusterSettings,
                 "source",
                 "target"
             );
@@ -144,6 +168,7 @@ public void testErrorCondition() {
                 clusterState,
                 (i) -> new DocsStats(between(10, 1000), between(1, 10), between(1, 10000)),
                 new StoreStats(between(1, 10000), between(1, 10000)),
+                clusterSettings,
                 "source",
                 "target"
             );
@@ -173,6 +198,7 @@ public void testErrorCondition() {
             clusterState,
             (i) -> new DocsStats(between(1, 1000), between(1, 1000), between(0, 10000)),
             new StoreStats(between(1, 10000), between(1, 10000)),
+            clusterSettings,
             "source",
             "target"
         );
@@ -189,7 +215,7 @@ public void testPassNumRoutingShards() {
             EmptyClusterInfoService.INSTANCE,
             EmptySnapshotsInfoService.INSTANCE
         );
-
+        ClusterSettings clusterSettings = createClusterSettings(CompatibilityMode.STRICT, RemoteStoreNodeService.Direction.NONE);
         RoutingTable routingTable = service.reroute(clusterState, "reroute").routingTable();
         clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
         // now we start the shard
@@ -204,6 +230,7 @@ public void testPassNumRoutingShards() {
             clusterState,
             null,
             new StoreStats(between(1, 10000), between(1, 10000)),
+            clusterSettings,
             "source",
             "target"
         );
@@ -217,6 +244,7 @@ public void testPassNumRoutingShards() {
             clusterState,
             null,
             new StoreStats(between(1, 10000), between(1, 10000)),
+            clusterSettings,
             "source",
             "target"
         );
@@ -235,6 +263,7 @@ public void testPassNumRoutingShardsAndFail() {
             EmptySnapshotsInfoService.INSTANCE
         );
 
+        ClusterSettings clusterSettings = createClusterSettings(CompatibilityMode.STRICT, RemoteStoreNodeService.Direction.NONE);
         RoutingTable routingTable = service.reroute(clusterState, "reroute").routingTable();
         clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
         // now we start the shard
@@ -249,6 +278,7 @@ public void testPassNumRoutingShardsAndFail() {
             clusterState,
             null,
             new StoreStats(between(1, 10000), between(1, 10000)),
+            clusterSettings,
             "source",
             "target"
         );
@@ -265,6 +295,7 @@ public void testPassNumRoutingShardsAndFail() {
                 finalState,
                 null,
                 new StoreStats(between(1, 10000), between(1, 10000)),
+                clusterSettings,
                 "source",
                 "target"
             )
@@ -286,6 +317,7 @@ public void testShrinkIndexSettings() {
             EmptySnapshotsInfoService.INSTANCE
         );
 
+        ClusterSettings clusterSettings = createClusterSettings(CompatibilityMode.STRICT, RemoteStoreNodeService.Direction.NONE);
         RoutingTable routingTable = service.reroute(clusterState, "reroute").routingTable();
         clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
         // now we start the shard
@@ -301,6 +333,7 @@ public void testShrinkIndexSettings() {
             clusterState,
             (i) -> stats,
             new StoreStats(between(1, 10000), between(1, 10000)),
+            clusterSettings,
             indexName,
             "target"
         );
@@ -325,6 +358,8 @@ public void testShrinkWithMaxShardSize() {
             EmptyClusterInfoService.INSTANCE,
             EmptySnapshotsInfoService.INSTANCE
         );
+
+        ClusterSettings clusterSettings = createClusterSettings(CompatibilityMode.STRICT, RemoteStoreNodeService.Direction.NONE);
         RoutingTable routingTable = service.reroute(clusterState, "reroute").routingTable();
         clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
         // now we start the shard
@@ -345,6 +380,7 @@ public void testShrinkWithMaxShardSize() {
             clusterState,
             (i) -> stats,
             new StoreStats(100, between(1, 10000)),
+            clusterSettings,
             indexName,
             "target"
         );
@@ -366,6 +402,7 @@ public void testShrinkWithMaxShardSize() {
             clusterState,
             (i) -> stats,
             new StoreStats(100, between(1, 10000)),
+            clusterSettings,
             indexName,
             "target"
         );
@@ -387,6 +424,7 @@ public void testShrinkWithMaxShardSize() {
             clusterState,
             (i) -> stats,
             new StoreStats(100, between(1, 10000)),
+            clusterSettings,
             indexName,
             "target"
         );
@@ -477,6 +515,7 @@ public void testIndexBlocks() {
             createClusterState(indexName, 10, 0, 40, Settings.builder().put("index.blocks.read_only", true).build())
         ).nodes(DiscoveryNodes.builder().add(newNode("node1"))).build();
 
+        ClusterSettings clusterSettings = createClusterSettings(CompatibilityMode.STRICT, RemoteStoreNodeService.Direction.NONE);
         // Target index will be blocked by [index.blocks.read_only=true] copied from the source index
         ResizeRequest resizeRequest = new ResizeRequest("target", indexName);
         ResizeType resizeType;
@@ -500,6 +539,7 @@ public void testIndexBlocks() {
                 finalState,
                 null,
                 new StoreStats(between(1, 10000), between(1, 10000)),
+                clusterSettings,
                 indexName,
                 "target"
             )
@@ -551,6 +591,7 @@ public void testIndexBlocks() {
             clusterState,
             (i) -> stats,
             new StoreStats(100, between(1, 10000)),
+            clusterSettings,
             indexName,
             "target"
         );
@@ -561,6 +602,127 @@ public void testIndexBlocks() {
         assertEquals(request.waitForActiveShards(), activeShardCount);
     }
 
+    public void testResizeFailuresDuringMigration() {
+        // We will keep all other settings correct for resize request,
+        // So we only need to test for the failures due to cluster setting validation while migration
+        final Settings directionEnabledNodeSettings = Settings.builder().put(REMOTE_STORE_MIGRATION_EXPERIMENTAL, "true").build();
+        FeatureFlags.initializeFeatureFlags(directionEnabledNodeSettings);
+        boolean isRemoteStoreEnabled = randomBoolean();
+        CompatibilityMode compatibilityMode = randomFrom(CompatibilityMode.values());
+        RemoteStoreNodeService.Direction migrationDirection = randomFrom(RemoteStoreNodeService.Direction.values());
+        // If not mixed mode, then migration direction is NONE.
+        if (!compatibilityMode.equals(CompatibilityMode.MIXED)) {
+            migrationDirection = RemoteStoreNodeService.Direction.NONE;
+        }
+        ClusterSettings clusterSettings = createClusterSettings(compatibilityMode, migrationDirection);
+
+        ClusterState clusterState = ClusterState.builder(
+            createClusterState(
+                "source",
+                10,
+                0,
+                40,
+                Settings.builder().put("index.blocks.write", true).put(SETTING_REMOTE_STORE_ENABLED, isRemoteStoreEnabled).build()
+            )
+        ).nodes(DiscoveryNodes.builder().add(newNode("node1"))).build();
+        AllocationService service = new AllocationService(
+            new AllocationDeciders(Collections.singleton(new MaxRetryAllocationDecider())),
+            new TestGatewayAllocator(),
+            new BalancedShardsAllocator(Settings.EMPTY),
+            EmptyClusterInfoService.INSTANCE,
+            EmptySnapshotsInfoService.INSTANCE
+        );
+
+        RoutingTable routingTable = service.reroute(clusterState, "reroute").routingTable();
+        clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
+        // now we start the shard
+        routingTable = OpenSearchAllocationTestCase.startInitializingShardsAndReroute(service, clusterState, "source").routingTable();
+        clusterState = ClusterState.builder(clusterState).routingTable(routingTable).build();
+        DocsStats stats = new DocsStats(between(0, (IndexWriter.MAX_DOCS) / 10), between(1, 1000), between(1, 10000));
+        ResizeRequest resizeRequest = new ResizeRequest("target", "source");
+        ResizeType resizeType;
+        int expectedShardsNum;
+        String cause;
+        switch (randomIntBetween(0, 2)) {
+            case 0:
+                resizeType = ResizeType.SHRINK;
+                expectedShardsNum = 5;
+                cause = "shrink_index";
+                break;
+            case 1:
+                resizeType = ResizeType.SPLIT;
+                expectedShardsNum = 20;
+                cause = "split_index";
+                break;
+            default:
+                resizeType = ResizeType.CLONE;
+                expectedShardsNum = 10;
+                cause = "clone_index";
+        }
+        resizeRequest.setResizeType(resizeType);
+        resizeRequest.getTargetIndexRequest()
+            .settings(Settings.builder().put("index.number_of_shards", expectedShardsNum).put("index.blocks.read_only", false).build());
+        final ActiveShardCount activeShardCount = randomBoolean() ? ActiveShardCount.ALL : ActiveShardCount.ONE;
+        resizeRequest.setWaitForActiveShards(activeShardCount);
+
+        if (compatibilityMode == CompatibilityMode.MIXED) {
+            if ((migrationDirection == RemoteStoreNodeService.Direction.REMOTE_STORE && isRemoteStoreEnabled == false)
+                || migrationDirection == RemoteStoreNodeService.Direction.DOCREP && isRemoteStoreEnabled == true) {
+                ClusterState finalState = clusterState;
+                IllegalStateException ise = expectThrows(
+                    IllegalStateException.class,
+                    () -> TransportResizeAction.prepareCreateIndexRequest(
+                        resizeRequest,
+                        finalState,
+                        (i) -> stats,
+                        new StoreStats(between(1, 10000), between(1, 10000)),
+                        clusterSettings,
+                        "source",
+                        "target"
+                    )
+                );
+                assertEquals(
+                    ise.getMessage(),
+                    "Index "
+                        + resizeType
+                        + " is not allowed as remote migration mode is mixed"
+                        + " and index is remote store "
+                        + (isRemoteStoreEnabled ? "enabled" : "disabled")
+                );
+            } else {
+                CreateIndexClusterStateUpdateRequest request = TransportResizeAction.prepareCreateIndexRequest(
+                    resizeRequest,
+                    clusterState,
+                    (i) -> stats,
+                    new StoreStats(100, between(1, 10000)),
+                    clusterSettings,
+                    "source",
+                    "target"
+                );
+                assertNotNull(request.recoverFrom());
+                assertEquals("source", request.recoverFrom().getName());
+                assertEquals(String.valueOf(expectedShardsNum), request.settings().get("index.number_of_shards"));
+                assertEquals(cause, request.cause());
+                assertEquals(request.waitForActiveShards(), activeShardCount);
+            }
+        } else {
+            CreateIndexClusterStateUpdateRequest request = TransportResizeAction.prepareCreateIndexRequest(
+                resizeRequest,
+                clusterState,
+                (i) -> stats,
+                new StoreStats(100, between(1, 10000)),
+                clusterSettings,
+                "source",
+                "target"
+            );
+            assertNotNull(request.recoverFrom());
+            assertEquals("source", request.recoverFrom().getName());
+            assertEquals(String.valueOf(expectedShardsNum), request.settings().get("index.number_of_shards"));
+            assertEquals(cause, request.cause());
+            assertEquals(request.waitForActiveShards(), activeShardCount);
+        }
+    }
+
     private DiscoveryNode newNode(String nodeId) {
         final Set<DiscoveryNodeRole> roles = Collections.unmodifiableSet(
             new HashSet<>(Arrays.asList(DiscoveryNodeRole.CLUSTER_MANAGER_ROLE, DiscoveryNodeRole.DATA_ROLE))
diff --git a/server/src/test/java/org/opensearch/action/bulk/BulkRequestParserTests.java b/server/src/test/java/org/opensearch/action/bulk/BulkRequestParserTests.java
index 4f07c098b0869..9625475607911 100644
--- a/server/src/test/java/org/opensearch/action/bulk/BulkRequestParserTests.java
+++ b/server/src/test/java/org/opensearch/action/bulk/BulkRequestParserTests.java
@@ -112,6 +112,13 @@ public void testUpdateRequest() throws IOException {
         parser.parse(request, "foo", null, null, null, true, false, MediaTypeRegistry.JSON, req -> fail(), updateRequest -> {
             assertFalse(updateRequest.isRequireAlias());
         }, req -> fail());
+
+        request = new BytesArray(
+            "{ \"update\":{ \"_id\": \"bar\", \"require_alias\": false, \"pipeline\": \"testPipeline\" } }\n{\"upsert\": {\"x\": 1}}\n"
+        );
+        parser.parse(request, "foo", null, null, null, true, false, MediaTypeRegistry.JSON, req -> fail(), updateRequest -> {
+            assertEquals(updateRequest.upsertRequest().getPipeline(), "testPipeline");
+        }, req -> fail());
     }
 
     public void testBarfOnLackOfTrailingNewline() {
diff --git a/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionIngestTests.java b/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionIngestTests.java
index 847140e81e201..a94a5d60b3f5a 100644
--- a/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionIngestTests.java
+++ b/server/src/test/java/org/opensearch/action/bulk/TransportBulkActionIngestTests.java
@@ -346,7 +346,8 @@ public void testIngestLocal() throws Exception {
             failureHandler.capture(),
             completionHandler.capture(),
             any(),
-            eq(Names.WRITE)
+            eq(Names.WRITE),
+            eq(bulkRequest)
         );
         completionHandler.getValue().accept(null, exception);
         assertTrue(failureCalled.get());
@@ -383,7 +384,8 @@ public void testSingleItemBulkActionIngestLocal() throws Exception {
             failureHandler.capture(),
             completionHandler.capture(),
             any(),
-            eq(Names.WRITE)
+            eq(Names.WRITE),
+            any()
         );
         completionHandler.getValue().accept(null, exception);
         assertTrue(failureCalled.get());
@@ -429,7 +431,8 @@ public void testIngestSystemLocal() throws Exception {
             failureHandler.capture(),
             completionHandler.capture(),
             any(),
-            eq(Names.SYSTEM_WRITE)
+            eq(Names.SYSTEM_WRITE),
+            eq(bulkRequest)
         );
         completionHandler.getValue().accept(null, exception);
         assertTrue(failureCalled.get());
@@ -460,7 +463,7 @@ public void testIngestForward() throws Exception {
         action.execute(null, bulkRequest, listener);
 
         // should not have executed ingest locally
-        verify(ingestService, never()).executeBulkRequest(anyInt(), any(), any(), any(), any(), any());
+        verify(ingestService, never()).executeBulkRequest(anyInt(), any(), any(), any(), any(), any(), any());
         // but instead should have sent to a remote node with the transport service
         ArgumentCaptor<DiscoveryNode> node = ArgumentCaptor.forClass(DiscoveryNode.class);
         verify(transportService).sendRequest(node.capture(), eq(BulkAction.NAME), any(), remoteResponseHandler.capture());
@@ -500,7 +503,7 @@ public void testSingleItemBulkActionIngestForward() throws Exception {
         singleItemBulkWriteAction.execute(null, indexRequest, listener);
 
         // should not have executed ingest locally
-        verify(ingestService, never()).executeBulkRequest(anyInt(), any(), any(), any(), any(), any());
+        verify(ingestService, never()).executeBulkRequest(anyInt(), any(), any(), any(), any(), any(), any());
         // but instead should have sent to a remote node with the transport service
         ArgumentCaptor<DiscoveryNode> node = ArgumentCaptor.forClass(DiscoveryNode.class);
         verify(transportService).sendRequest(node.capture(), eq(BulkAction.NAME), any(), remoteResponseHandler.capture());
@@ -586,7 +589,8 @@ private void validatePipelineWithBulkUpsert(@Nullable String indexRequestIndexNa
             failureHandler.capture(),
             completionHandler.capture(),
             any(),
-            eq(Names.WRITE)
+            eq(Names.WRITE),
+            eq(bulkRequest)
         );
         assertEquals(indexRequest1.getPipeline(), "default_pipeline");
         assertEquals(indexRequest2.getPipeline(), "default_pipeline");
@@ -629,7 +633,8 @@ public void testDoExecuteCalledTwiceCorrectly() throws Exception {
             failureHandler.capture(),
             completionHandler.capture(),
             any(),
-            eq(Names.WRITE)
+            eq(Names.WRITE),
+            any()
         );
         completionHandler.getValue().accept(null, exception);
         assertFalse(action.indexCreated); // still no index yet, the ingest node failed.
@@ -716,7 +721,8 @@ public void testFindDefaultPipelineFromTemplateMatch() {
             failureHandler.capture(),
             completionHandler.capture(),
             any(),
-            eq(Names.WRITE)
+            eq(Names.WRITE),
+            any()
         );
     }
 
@@ -755,7 +761,8 @@ public void testFindDefaultPipelineFromV2TemplateMatch() {
             failureHandler.capture(),
             completionHandler.capture(),
             any(),
-            eq(Names.WRITE)
+            eq(Names.WRITE),
+            any()
         );
     }
 
@@ -780,7 +787,8 @@ private void validateDefaultPipeline(IndexRequest indexRequest) {
             failureHandler.capture(),
             completionHandler.capture(),
             any(),
-            eq(Names.WRITE)
+            eq(Names.WRITE),
+            any()
         );
         assertEquals(indexRequest.getPipeline(), "default_pipeline");
         completionHandler.getValue().accept(null, exception);
diff --git a/server/src/test/java/org/opensearch/action/bulk/TransportShardBulkActionTests.java b/server/src/test/java/org/opensearch/action/bulk/TransportShardBulkActionTests.java
index 65b555649b2d0..6331861c3dcb9 100644
--- a/server/src/test/java/org/opensearch/action/bulk/TransportShardBulkActionTests.java
+++ b/server/src/test/java/org/opensearch/action/bulk/TransportShardBulkActionTests.java
@@ -107,6 +107,7 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.LongSupplier;
 
+import static org.opensearch.index.remote.RemoteStoreTestsHelper.createIndexSettings;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.CoreMatchers.instanceOf;
 import static org.hamcrest.CoreMatchers.not;
@@ -1237,14 +1238,14 @@ public void testHandlePrimaryTermValidationRequestSuccess() {
     public void testGetReplicationModeWithRemoteTranslog() {
         TransportShardBulkAction action = createAction();
         final IndexShard indexShard = mock(IndexShard.class);
-        when(indexShard.isRemoteTranslogEnabled()).thenReturn(true);
+        when(indexShard.indexSettings()).thenReturn(createIndexSettings(true));
         assertEquals(ReplicationMode.PRIMARY_TERM_VALIDATION, action.getReplicationMode(indexShard));
     }
 
     public void testGetReplicationModeWithLocalTranslog() {
         TransportShardBulkAction action = createAction();
         final IndexShard indexShard = mock(IndexShard.class);
-        when(indexShard.isRemoteTranslogEnabled()).thenReturn(false);
+        when(indexShard.indexSettings()).thenReturn(createIndexSettings(false));
         assertEquals(ReplicationMode.FULL_REPLICATION, action.getReplicationMode(indexShard));
     }
 
diff --git a/server/src/test/java/org/opensearch/action/resync/TransportResyncReplicationActionTests.java b/server/src/test/java/org/opensearch/action/resync/TransportResyncReplicationActionTests.java
index da87a0a967f53..a2fefd6278321 100644
--- a/server/src/test/java/org/opensearch/action/resync/TransportResyncReplicationActionTests.java
+++ b/server/src/test/java/org/opensearch/action/resync/TransportResyncReplicationActionTests.java
@@ -84,6 +84,7 @@
 import static java.util.Collections.emptyList;
 import static java.util.Collections.emptyMap;
 import static org.opensearch.action.support.replication.ClusterStateCreationUtils.state;
+import static org.opensearch.index.remote.RemoteStoreTestsHelper.createIndexSettings;
 import static org.opensearch.test.ClusterServiceUtils.createClusterService;
 import static org.opensearch.test.ClusterServiceUtils.setState;
 import static org.opensearch.transport.TransportService.NOOP_TRANSPORT_INTERCEPTOR;
@@ -233,14 +234,14 @@ public void testResyncDoesNotBlockOnPrimaryAction() throws Exception {
     public void testGetReplicationModeWithRemoteTranslog() {
         final TransportResyncReplicationAction action = createAction();
         final IndexShard indexShard = mock(IndexShard.class);
-        when(indexShard.isRemoteTranslogEnabled()).thenReturn(true);
+        when(indexShard.indexSettings()).thenReturn(createIndexSettings(true));
         assertEquals(ReplicationMode.NO_REPLICATION, action.getReplicationMode(indexShard));
     }
 
     public void testGetReplicationModeWithLocalTranslog() {
         final TransportResyncReplicationAction action = createAction();
         final IndexShard indexShard = mock(IndexShard.class);
-        when(indexShard.isRemoteTranslogEnabled()).thenReturn(false);
+        when(indexShard.indexSettings()).thenReturn(createIndexSettings(false));
         assertEquals(ReplicationMode.FULL_REPLICATION, action.getReplicationMode(indexShard));
     }
 
diff --git a/server/src/test/java/org/opensearch/action/search/AbstractSearchAsyncActionTests.java b/server/src/test/java/org/opensearch/action/search/AbstractSearchAsyncActionTests.java
index 420289d3ff2e5..7dcbf213d6c9d 100644
--- a/server/src/test/java/org/opensearch/action/search/AbstractSearchAsyncActionTests.java
+++ b/server/src/test/java/org/opensearch/action/search/AbstractSearchAsyncActionTests.java
@@ -70,6 +70,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.EnumSet;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
@@ -442,6 +443,24 @@ public void testShardNotAvailableWithDisallowPartialFailures() {
         assertEquals(0, searchPhaseExecutionException.getSuppressed().length);
     }
 
+    public void testShardNotAvailableWithIgnoreUnavailable() {
+        SearchRequest searchRequest = new SearchRequest().allowPartialSearchResults(false)
+            .indicesOptions(new IndicesOptions(EnumSet.of(IndicesOptions.Option.IGNORE_UNAVAILABLE), IndicesOptions.WildcardStates.NONE));
+        AtomicReference<Exception> exception = new AtomicReference<>();
+        ActionListener<SearchResponse> listener = ActionListener.wrap(response -> {}, exception::set);
+        int numShards = randomIntBetween(2, 10);
+        ArraySearchPhaseResults<SearchPhaseResult> phaseResults = new ArraySearchPhaseResults<>(numShards);
+        AbstractSearchAsyncAction<SearchPhaseResult> action = createAction(searchRequest, phaseResults, listener, false, new AtomicLong());
+        // skip one to avoid the "all shards failed" failure.
+        SearchShardIterator skipIterator = new SearchShardIterator(null, null, Collections.emptyList(), null);
+        skipIterator.resetAndSkip();
+        action.skipShard(skipIterator);
+
+        // Validate no exception is thrown
+        action.executeNextPhase(action, createFetchSearchPhase());
+        action.sendSearchResponse(InternalSearchResponse.empty(), phaseResults.results);
+    }
+
     private static ArraySearchPhaseResults<SearchPhaseResult> phaseResults(
         Set<ShardSearchContextId> contextIds,
         List<Tuple<String, String>> nodeLookups,
diff --git a/server/src/test/java/org/opensearch/action/search/SearchRequestTests.java b/server/src/test/java/org/opensearch/action/search/SearchRequestTests.java
index 9ee314e77ca7e..40514c526f190 100644
--- a/server/src/test/java/org/opensearch/action/search/SearchRequestTests.java
+++ b/server/src/test/java/org/opensearch/action/search/SearchRequestTests.java
@@ -46,6 +46,7 @@
 import org.opensearch.search.Scroll;
 import org.opensearch.search.builder.PointInTimeBuilder;
 import org.opensearch.search.builder.SearchSourceBuilder;
+import org.opensearch.search.fetch.subphase.FetchSourceContext;
 import org.opensearch.search.rescore.QueryRescorerBuilder;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.test.VersionUtils;
@@ -76,6 +77,35 @@ protected SearchRequest createSearchRequest() throws IOException {
         );
     }
 
+    public void testClone() throws IOException {
+        SearchRequest searchRequest = new SearchRequest();
+        SearchRequest clonedRequest = searchRequest.deepCopy();
+        assertEquals(searchRequest.hashCode(), clonedRequest.hashCode());
+        assertNotSame(searchRequest, clonedRequest);
+
+        String[] includes = new String[] { "field1.*" };
+        String[] excludes = new String[] { "field2.*" };
+        FetchSourceContext fetchSourceContext = new FetchSourceContext(true, includes, excludes);
+        SearchSourceBuilder source = new SearchSourceBuilder().fetchSource(fetchSourceContext);
+        SearchRequest complexSearchRequest = createSearchRequest().source(source);
+        complexSearchRequest.requestCache(false);
+        complexSearchRequest.scroll(new TimeValue(1000));
+        SearchRequest clonedComplexRequest = complexSearchRequest.deepCopy();
+        assertEquals(complexSearchRequest.hashCode(), clonedComplexRequest.hashCode());
+        assertNotSame(complexSearchRequest, clonedComplexRequest);
+        assertEquals(fetchSourceContext, clonedComplexRequest.source().fetchSource());
+        assertNotSame(fetchSourceContext, clonedComplexRequest.source().fetchSource());
+        // Change the value of the original includes array and excludes array
+        includes[0] = "new_field1.*";
+        excludes[0] = "new_field2.*";
+        // Values in the original fetchSource object should be updated
+        assertEquals("new_field1.*", complexSearchRequest.source().fetchSource().includes()[0]);
+        assertEquals("new_field2.*", complexSearchRequest.source().fetchSource().excludes()[0]);
+        // Values in the cloned fetchSource object should not be updated
+        assertEquals("field1.*", clonedComplexRequest.source().fetchSource().includes()[0]);
+        assertEquals("field2.*", clonedComplexRequest.source().fetchSource().excludes()[0]);
+    }
+
     public void testWithLocalReduction() {
         expectThrows(NullPointerException.class, () -> SearchRequest.subSearchRequest(null, Strings.EMPTY_ARRAY, "", 0, randomBoolean()));
         SearchRequest request = new SearchRequest();
diff --git a/server/src/test/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeActionTests.java b/server/src/test/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeActionTests.java
index 538416e1137f5..35c5c5e605b4d 100644
--- a/server/src/test/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeActionTests.java
+++ b/server/src/test/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeActionTests.java
@@ -16,11 +16,15 @@
 import org.opensearch.OpenSearchException;
 import org.opensearch.Version;
 import org.opensearch.action.ActionRequestValidationException;
+import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
+import org.opensearch.action.admin.cluster.settings.TransportClusterUpdateSettingsAction;
 import org.opensearch.action.support.ActionFilters;
 import org.opensearch.action.support.PlainActionFuture;
 import org.opensearch.action.support.ThreadedActionListener;
 import org.opensearch.action.support.replication.ClusterStateCreationUtils;
+import org.opensearch.cluster.ClusterName;
 import org.opensearch.cluster.ClusterState;
+import org.opensearch.cluster.EmptyClusterInfoService;
 import org.opensearch.cluster.NotClusterManagerException;
 import org.opensearch.cluster.block.ClusterBlock;
 import org.opensearch.cluster.block.ClusterBlockException;
@@ -28,14 +32,22 @@
 import org.opensearch.cluster.block.ClusterBlocks;
 import org.opensearch.cluster.coordination.FailedToCommitClusterStateException;
 import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
+import org.opensearch.cluster.metadata.Metadata;
 import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.cluster.node.DiscoveryNodeRole;
 import org.opensearch.cluster.node.DiscoveryNodes;
+import org.opensearch.cluster.routing.allocation.AllocationService;
+import org.opensearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
+import org.opensearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.opensearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider;
 import org.opensearch.cluster.service.ClusterManagerThrottlingException;
 import org.opensearch.cluster.service.ClusterService;
+import org.opensearch.common.UUIDs;
 import org.opensearch.common.action.ActionFuture;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.common.settings.SettingsException;
 import org.opensearch.common.unit.TimeValue;
+import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.common.util.concurrent.ThreadContext;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.action.ActionResponse;
@@ -44,9 +56,12 @@
 import org.opensearch.core.rest.RestStatus;
 import org.opensearch.discovery.ClusterManagerNotDiscoveredException;
 import org.opensearch.node.NodeClosedException;
+import org.opensearch.node.remotestore.RemoteStoreNodeService;
+import org.opensearch.snapshots.EmptySnapshotsInfoService;
 import org.opensearch.tasks.Task;
 import org.opensearch.telemetry.tracing.noop.NoopTracer;
 import org.opensearch.test.OpenSearchTestCase;
+import org.opensearch.test.gateway.TestGatewayAllocator;
 import org.opensearch.test.transport.CapturingTransport;
 import org.opensearch.threadpool.TestThreadPool;
 import org.opensearch.threadpool.ThreadPool;
@@ -59,7 +74,9 @@
 
 import java.io.IOException;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.BrokenBarrierException;
@@ -68,8 +85,15 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import static org.opensearch.common.util.FeatureFlags.REMOTE_STORE_MIGRATION_EXPERIMENTAL;
+import static org.opensearch.index.remote.RemoteMigrationIndexMetadataUpdaterTests.createIndexMetadataWithDocrepSettings;
+import static org.opensearch.index.remote.RemoteMigrationIndexMetadataUpdaterTests.createIndexMetadataWithRemoteStoreSettings;
+import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY;
+import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY;
+import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING;
 import static org.opensearch.test.ClusterServiceUtils.createClusterService;
 import static org.opensearch.test.ClusterServiceUtils.setState;
+import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.instanceOf;
 
@@ -692,4 +716,277 @@ protected void masterOperation(Task task, Request request, ClusterState state, A
         assertFalse(retried.get());
         assertFalse(exception.get());
     }
+
+    public void testDontAllowSwitchingToStrictCompatibilityModeForMixedCluster() {
+        Settings nodeSettings = Settings.builder().put(REMOTE_STORE_MIGRATION_EXPERIMENTAL, "true").build();
+        FeatureFlags.initializeFeatureFlags(nodeSettings);
+
+        // request to change cluster compatibility mode to STRICT
+        Settings currentCompatibilityModeSettings = Settings.builder()
+            .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), RemoteStoreNodeService.CompatibilityMode.MIXED)
+            .build();
+        Settings intendedCompatibilityModeSettings = Settings.builder()
+            .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), RemoteStoreNodeService.CompatibilityMode.STRICT)
+            .build();
+        ClusterUpdateSettingsRequest request = new ClusterUpdateSettingsRequest();
+        request.persistentSettings(intendedCompatibilityModeSettings);
+
+        // mixed cluster (containing both remote and non-remote nodes)
+        DiscoveryNode nonRemoteNode1 = new DiscoveryNode(UUIDs.base64UUID(), buildNewFakeTransportAddress(), Version.CURRENT);
+        DiscoveryNode remoteNode1 = new DiscoveryNode(
+            UUIDs.base64UUID(),
+            buildNewFakeTransportAddress(),
+            getRemoteStoreNodeAttributes(),
+            DiscoveryNodeRole.BUILT_IN_ROLES,
+            Version.CURRENT
+        );
+
+        DiscoveryNodes discoveryNodes = DiscoveryNodes.builder()
+            .add(nonRemoteNode1)
+            .localNodeId(nonRemoteNode1.getId())
+            .add(remoteNode1)
+            .localNodeId(remoteNode1.getId())
+            .build();
+
+        Metadata metadata = Metadata.builder().persistentSettings(currentCompatibilityModeSettings).build();
+
+        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).metadata(metadata).nodes(discoveryNodes).build();
+        AllocationService allocationService = new AllocationService(
+            new AllocationDeciders(Collections.singleton(new MaxRetryAllocationDecider())),
+            new TestGatewayAllocator(),
+            new BalancedShardsAllocator(Settings.EMPTY),
+            EmptyClusterInfoService.INSTANCE,
+            EmptySnapshotsInfoService.INSTANCE
+        );
+        TransportClusterUpdateSettingsAction transportClusterUpdateSettingsAction = new TransportClusterUpdateSettingsAction(
+            transportService,
+            clusterService,
+            threadPool,
+            allocationService,
+            new ActionFilters(Collections.emptySet()),
+            new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)),
+            clusterService.getClusterSettings()
+        );
+
+        final SettingsException exception = expectThrows(
+            SettingsException.class,
+            () -> transportClusterUpdateSettingsAction.validateCompatibilityModeSettingRequest(request, clusterState)
+        );
+        assertEquals(
+            "can not switch to STRICT compatibility mode when the cluster contains both remote and non-remote nodes",
+            exception.getMessage()
+        );
+
+        DiscoveryNode nonRemoteNode2 = new DiscoveryNode(UUIDs.base64UUID(), buildNewFakeTransportAddress(), Version.CURRENT);
+        DiscoveryNode remoteNode2 = new DiscoveryNode(
+            UUIDs.base64UUID(),
+            buildNewFakeTransportAddress(),
+            getRemoteStoreNodeAttributes(),
+            DiscoveryNodeRole.BUILT_IN_ROLES,
+            Version.CURRENT
+        );
+
+        // cluster with only non-remote nodes
+        discoveryNodes = DiscoveryNodes.builder()
+            .add(nonRemoteNode1)
+            .localNodeId(nonRemoteNode1.getId())
+            .add(nonRemoteNode2)
+            .localNodeId(nonRemoteNode2.getId())
+            .build();
+
+        metadata = createIndexMetadataWithRemoteStoreSettings("test-index");
+        ClusterState sameTypeClusterState = ClusterState.builder(clusterState).nodes(discoveryNodes).metadata(metadata).build();
+        transportClusterUpdateSettingsAction.validateCompatibilityModeSettingRequest(request, sameTypeClusterState);
+
+        // cluster with only non-remote nodes
+        discoveryNodes = DiscoveryNodes.builder()
+            .add(remoteNode1)
+            .localNodeId(remoteNode1.getId())
+            .add(remoteNode2)
+            .localNodeId(remoteNode2.getId())
+            .build();
+        sameTypeClusterState = ClusterState.builder(sameTypeClusterState).nodes(discoveryNodes).metadata(metadata).build();
+        transportClusterUpdateSettingsAction.validateCompatibilityModeSettingRequest(request, sameTypeClusterState);
+    }
+
+    public void testDontAllowSwitchingToStrictCompatibilityModeWithoutRemoteIndexSettings() {
+        Settings nodeSettings = Settings.builder().put(REMOTE_STORE_MIGRATION_EXPERIMENTAL, "true").build();
+        FeatureFlags.initializeFeatureFlags(nodeSettings);
+        Settings currentCompatibilityModeSettings = Settings.builder()
+            .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), RemoteStoreNodeService.CompatibilityMode.MIXED)
+            .build();
+        Settings intendedCompatibilityModeSettings = Settings.builder()
+            .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), RemoteStoreNodeService.CompatibilityMode.STRICT)
+            .build();
+        ClusterUpdateSettingsRequest request = new ClusterUpdateSettingsRequest();
+        request.persistentSettings(intendedCompatibilityModeSettings);
+        DiscoveryNode remoteNode1 = new DiscoveryNode(
+            UUIDs.base64UUID(),
+            buildNewFakeTransportAddress(),
+            getRemoteStoreNodeAttributes(),
+            DiscoveryNodeRole.BUILT_IN_ROLES,
+            Version.CURRENT
+        );
+        DiscoveryNode remoteNode2 = new DiscoveryNode(
+            UUIDs.base64UUID(),
+            buildNewFakeTransportAddress(),
+            getRemoteStoreNodeAttributes(),
+            DiscoveryNodeRole.BUILT_IN_ROLES,
+            Version.CURRENT
+        );
+        DiscoveryNodes discoveryNodes = DiscoveryNodes.builder()
+            .add(remoteNode1)
+            .localNodeId(remoteNode1.getId())
+            .add(remoteNode2)
+            .localNodeId(remoteNode2.getId())
+            .build();
+        AllocationService allocationService = new AllocationService(
+            new AllocationDeciders(Collections.singleton(new MaxRetryAllocationDecider())),
+            new TestGatewayAllocator(),
+            new BalancedShardsAllocator(Settings.EMPTY),
+            EmptyClusterInfoService.INSTANCE,
+            EmptySnapshotsInfoService.INSTANCE
+        );
+        TransportClusterUpdateSettingsAction transportClusterUpdateSettingsAction = new TransportClusterUpdateSettingsAction(
+            transportService,
+            clusterService,
+            threadPool,
+            allocationService,
+            new ActionFilters(Collections.emptySet()),
+            new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)),
+            clusterService.getClusterSettings()
+        );
+
+        Metadata nonRemoteIndexMd = Metadata.builder(createIndexMetadataWithDocrepSettings("test"))
+            .persistentSettings(currentCompatibilityModeSettings)
+            .build();
+        final ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
+            .metadata(nonRemoteIndexMd)
+            .nodes(discoveryNodes)
+            .build();
+        final SettingsException exception = expectThrows(
+            SettingsException.class,
+            () -> transportClusterUpdateSettingsAction.validateCompatibilityModeSettingRequest(request, clusterState)
+        );
+        assertEquals(
+            "can not switch to STRICT compatibility mode since all indices in the cluster does not have remote store based index settings",
+            exception.getMessage()
+        );
+
+        Metadata remoteIndexMd = Metadata.builder(createIndexMetadataWithRemoteStoreSettings("test"))
+            .persistentSettings(currentCompatibilityModeSettings)
+            .build();
+        ClusterState clusterStateWithRemoteIndices = ClusterState.builder(ClusterName.DEFAULT)
+            .metadata(remoteIndexMd)
+            .nodes(discoveryNodes)
+            .build();
+        transportClusterUpdateSettingsAction.validateCompatibilityModeSettingRequest(request, clusterStateWithRemoteIndices);
+    }
+
+    public void testDontAllowSwitchingCompatibilityModeForClusterWithMultipleVersions() {
+        Settings nodeSettings = Settings.builder().put(REMOTE_STORE_MIGRATION_EXPERIMENTAL, "true").build();
+        FeatureFlags.initializeFeatureFlags(nodeSettings);
+
+        // request to change cluster compatibility mode
+        boolean toStrictMode = randomBoolean();
+        Settings currentCompatibilityModeSettings = Settings.builder()
+            .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), RemoteStoreNodeService.CompatibilityMode.MIXED)
+            .build();
+        Settings intendedCompatibilityModeSettings = Settings.builder()
+            .put(
+                REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(),
+                toStrictMode ? RemoteStoreNodeService.CompatibilityMode.STRICT : RemoteStoreNodeService.CompatibilityMode.MIXED
+            )
+            .build();
+        ClusterUpdateSettingsRequest request = new ClusterUpdateSettingsRequest();
+        request.persistentSettings(intendedCompatibilityModeSettings);
+
+        // two different but compatible open search versions for the discovery nodes
+        final Version version1 = Version.V_2_13_0;
+        final Version version2 = Version.V_2_13_1;
+
+        DiscoveryNode discoveryNode1 = new DiscoveryNode(
+            UUIDs.base64UUID(),
+            buildNewFakeTransportAddress(),
+            toStrictMode ? getRemoteStoreNodeAttributes() : Collections.emptyMap(),
+            DiscoveryNodeRole.BUILT_IN_ROLES,
+            version1
+        );
+        DiscoveryNode discoveryNode2 = new DiscoveryNode(
+            UUIDs.base64UUID(),
+            buildNewFakeTransportAddress(),
+            toStrictMode ? getRemoteStoreNodeAttributes() : Collections.emptyMap(),
+            DiscoveryNodeRole.BUILT_IN_ROLES,
+            version2 // not same as discoveryNode1
+        );
+
+        DiscoveryNodes discoveryNodes = DiscoveryNodes.builder()
+            .add(discoveryNode1)
+            .localNodeId(discoveryNode1.getId())
+            .add(discoveryNode2)
+            .localNodeId(discoveryNode2.getId())
+            .build();
+
+        Metadata metadata = Metadata.builder().persistentSettings(currentCompatibilityModeSettings).build();
+
+        ClusterState differentVersionClusterState = ClusterState.builder(ClusterName.DEFAULT)
+            .metadata(metadata)
+            .nodes(discoveryNodes)
+            .build();
+        AllocationService allocationService = new AllocationService(
+            new AllocationDeciders(Collections.singleton(new MaxRetryAllocationDecider())),
+            new TestGatewayAllocator(),
+            new BalancedShardsAllocator(Settings.EMPTY),
+            EmptyClusterInfoService.INSTANCE,
+            EmptySnapshotsInfoService.INSTANCE
+        );
+        TransportClusterUpdateSettingsAction transportClusterUpdateSettingsAction = new TransportClusterUpdateSettingsAction(
+            transportService,
+            clusterService,
+            threadPool,
+            allocationService,
+            new ActionFilters(Collections.emptySet()),
+            new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY)),
+            clusterService.getClusterSettings()
+        );
+
+        // changing compatibility mode when all nodes are not of the same version
+        final SettingsException exception = expectThrows(
+            SettingsException.class,
+            () -> transportClusterUpdateSettingsAction.validateCompatibilityModeSettingRequest(request, differentVersionClusterState)
+        );
+        assertThat(
+            exception.getMessage(),
+            containsString("can not change the compatibility mode when all the nodes in cluster are not of the same version")
+        );
+
+        // changing compatibility mode when all nodes are of the same version
+        discoveryNode2 = new DiscoveryNode(
+            UUIDs.base64UUID(),
+            buildNewFakeTransportAddress(),
+            toStrictMode ? getRemoteStoreNodeAttributes() : Collections.emptyMap(),
+            DiscoveryNodeRole.BUILT_IN_ROLES,
+            version1 // same as discoveryNode1
+        );
+        discoveryNodes = DiscoveryNodes.builder()
+            .add(discoveryNode1)
+            .localNodeId(discoveryNode1.getId())
+            .add(discoveryNode2)
+            .localNodeId(discoveryNode2.getId())
+            .build();
+
+        ClusterState sameVersionClusterState = ClusterState.builder(differentVersionClusterState)
+            .nodes(discoveryNodes)
+            .metadata(createIndexMetadataWithRemoteStoreSettings("test"))
+            .build();
+        transportClusterUpdateSettingsAction.validateCompatibilityModeSettingRequest(request, sameVersionClusterState);
+    }
+
+    private Map<String, String> getRemoteStoreNodeAttributes() {
+        Map<String, String> remoteStoreNodeAttributes = new HashMap<>();
+        remoteStoreNodeAttributes.put(REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, "my-segment-repo-1");
+        remoteStoreNodeAttributes.put(REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, "my-translog-repo-1");
+        return remoteStoreNodeAttributes;
+    }
+
 }
diff --git a/server/src/test/java/org/opensearch/action/support/clustermanager/TransportClusterManagerTermCheckTests.java b/server/src/test/java/org/opensearch/action/support/clustermanager/TransportClusterManagerTermCheckTests.java
index 8c7b7a0940c82..6d118cbccb42d 100644
--- a/server/src/test/java/org/opensearch/action/support/clustermanager/TransportClusterManagerTermCheckTests.java
+++ b/server/src/test/java/org/opensearch/action/support/clustermanager/TransportClusterManagerTermCheckTests.java
@@ -33,10 +33,10 @@
 
 import org.opensearch.Version;
 import org.opensearch.action.ActionRequestValidationException;
-import org.opensearch.action.admin.cluster.state.term.GetTermVersionResponse;
 import org.opensearch.action.support.ActionFilters;
 import org.opensearch.action.support.PlainActionFuture;
 import org.opensearch.action.support.ThreadedActionListener;
+import org.opensearch.action.support.clustermanager.term.GetTermVersionResponse;
 import org.opensearch.action.support.replication.ClusterStateCreationUtils;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.block.ClusterBlockException;
@@ -223,7 +223,7 @@ public void testTermCheckMatchWithClusterManager() throws ExecutionException, In
         assertThat(transport.capturedRequests().length, equalTo(1));
         CapturingTransport.CapturedRequest capturedRequest = transport.capturedRequests()[0];
         assertTrue(capturedRequest.node.isClusterManagerNode());
-        assertThat(capturedRequest.action, equalTo("cluster:monitor/term"));
+        assertThat(capturedRequest.action, equalTo("internal:monitor/term"));
         GetTermVersionResponse response = new GetTermVersionResponse(
             new ClusterStateTermVersion(
                 clusterService.state().getClusterName(),
@@ -249,7 +249,7 @@ public void testTermCheckNoMatchWithClusterManager() throws ExecutionException,
         assertThat(transport.capturedRequests().length, equalTo(1));
         CapturingTransport.CapturedRequest termCheckRequest = transport.capturedRequests()[0];
         assertTrue(termCheckRequest.node.isClusterManagerNode());
-        assertThat(termCheckRequest.action, equalTo("cluster:monitor/term"));
+        assertThat(termCheckRequest.action, equalTo("internal:monitor/term"));
         GetTermVersionResponse noMatchResponse = new GetTermVersionResponse(
             new ClusterStateTermVersion(
                 clusterService.state().getClusterName(),
diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/state/term/ClusterTermVersionIT.java b/server/src/test/java/org/opensearch/action/support/clustermanager/term/ClusterTermVersionIT.java
similarity index 98%
rename from server/src/test/java/org/opensearch/action/admin/cluster/state/term/ClusterTermVersionIT.java
rename to server/src/test/java/org/opensearch/action/support/clustermanager/term/ClusterTermVersionIT.java
index fa2a6121af349..7b783e025a575 100644
--- a/server/src/test/java/org/opensearch/action/admin/cluster/state/term/ClusterTermVersionIT.java
+++ b/server/src/test/java/org/opensearch/action/support/clustermanager/term/ClusterTermVersionIT.java
@@ -6,7 +6,7 @@
  * compatible open source license.
  */
 
-package org.opensearch.action.admin.cluster.state.term;
+package org.opensearch.action.support.clustermanager.term;
 
 import org.opensearch.action.admin.cluster.state.ClusterStateAction;
 import org.opensearch.action.admin.cluster.state.ClusterStateRequest;
diff --git a/server/src/test/java/org/opensearch/action/admin/cluster/state/term/ClusterTermVersionTests.java b/server/src/test/java/org/opensearch/action/support/clustermanager/term/ClusterTermVersionTests.java
similarity index 93%
rename from server/src/test/java/org/opensearch/action/admin/cluster/state/term/ClusterTermVersionTests.java
rename to server/src/test/java/org/opensearch/action/support/clustermanager/term/ClusterTermVersionTests.java
index 22d9623eebdbe..23ae8c6a58776 100644
--- a/server/src/test/java/org/opensearch/action/admin/cluster/state/term/ClusterTermVersionTests.java
+++ b/server/src/test/java/org/opensearch/action/support/clustermanager/term/ClusterTermVersionTests.java
@@ -6,7 +6,7 @@
  * compatible open source license.
  */
 
-package org.opensearch.action.admin.cluster.state.term;
+package org.opensearch.action.support.clustermanager.term;
 
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.test.OpenSearchSingleNodeTestCase;
diff --git a/server/src/test/java/org/opensearch/action/support/replication/ReplicationModeAwareProxyTests.java b/server/src/test/java/org/opensearch/action/support/replication/ReplicationModeAwareProxyTests.java
new file mode 100644
index 0000000000000..626c2f74f09c4
--- /dev/null
+++ b/server/src/test/java/org/opensearch/action/support/replication/ReplicationModeAwareProxyTests.java
@@ -0,0 +1,216 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.action.support.replication;
+
+import org.opensearch.cluster.node.DiscoveryNodes;
+import org.opensearch.cluster.routing.AllocationId;
+import org.opensearch.cluster.routing.ShardRouting;
+import org.opensearch.cluster.routing.ShardRoutingState;
+import org.opensearch.cluster.routing.TestShardRouting;
+import org.opensearch.core.index.Index;
+import org.opensearch.core.index.shard.ShardId;
+import org.opensearch.index.shard.IndexShardTestUtils;
+import org.opensearch.test.OpenSearchTestCase;
+
+import static org.mockito.Mockito.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
+
+public class ReplicationModeAwareProxyTests extends OpenSearchTestCase {
+
+    /*
+    Replication action running on the same primary copy from which it originates.
+    Action should not run and proxy should return ReplicationMode.NO_REPLICATION
+     */
+    public void testDetermineReplicationModeTargetRoutingCurrentPrimary() {
+        ShardRouting targetRouting = TestShardRouting.newShardRouting(
+            new ShardId(new Index("test_index", "_na_"), 0),
+            "dummy-node",
+            null,
+            true,
+            ShardRoutingState.STARTED,
+            AllocationId.newInitializing("abc")
+        );
+        ShardRouting primaryRouting = TestShardRouting.newShardRouting(
+            new ShardId(new Index("test_index", "_na_"), 0),
+            "dummy-node",
+            null,
+            true,
+            ShardRoutingState.STARTED,
+            AllocationId.newInitializing("abc")
+        );
+        final ReplicationModeAwareProxy replicationModeAwareProxy = new ReplicationModeAwareProxy(
+            ReplicationMode.NO_REPLICATION,
+            DiscoveryNodes.builder().add(IndexShardTestUtils.getFakeRemoteEnabledNode("dummy-node")).build(),
+            mock(TransportReplicationAction.ReplicasProxy.class),
+            mock(TransportReplicationAction.ReplicasProxy.class),
+            randomBoolean()
+        );
+        assertEquals(ReplicationMode.NO_REPLICATION, replicationModeAwareProxy.determineReplicationMode(targetRouting, primaryRouting));
+    }
+
+    /*
+     Replication action originating from failing primary to replica being promoted to primary
+     Action should run and proxy should return ReplicationMode.FULL_REPLICATION
+     */
+    public void testDetermineReplicationModeTargetRoutingRelocatingPrimary() {
+        AllocationId primaryId = AllocationId.newRelocation(AllocationId.newInitializing());
+        AllocationId relocationTargetId = AllocationId.newTargetRelocation(primaryId);
+        ShardRouting targetRouting = TestShardRouting.newShardRouting(
+            new ShardId(new Index("test_index", "_na_"), 0),
+            "dummy-node-2",
+            null,
+            true,
+            ShardRoutingState.INITIALIZING,
+            relocationTargetId
+        );
+        ShardRouting primaryRouting = TestShardRouting.newShardRouting(
+            new ShardId(new Index("test_index", "_na_"), 0),
+            "dummy-node",
+            "dummy-node-2",
+            true,
+            ShardRoutingState.RELOCATING,
+            primaryId
+        );
+        final ReplicationModeAwareProxy replicationModeAwareProxy = new ReplicationModeAwareProxy(
+            ReplicationMode.NO_REPLICATION,
+            DiscoveryNodes.builder()
+                .add(IndexShardTestUtils.getFakeRemoteEnabledNode(targetRouting.currentNodeId()))
+                .add(IndexShardTestUtils.getFakeRemoteEnabledNode(primaryRouting.currentNodeId()))
+                .build(),
+            mock(TransportReplicationAction.ReplicasProxy.class),
+            mock(TransportReplicationAction.ReplicasProxy.class),
+            randomBoolean()
+        );
+        assertEquals(ReplicationMode.FULL_REPLICATION, replicationModeAwareProxy.determineReplicationMode(targetRouting, primaryRouting));
+    }
+
+    /*
+     Replication action originating from remote enabled primary to docrep replica during remote store migration
+     Action should run and proxy should return ReplicationMode.FULL_REPLICATION
+     */
+    public void testDetermineReplicationModeTargetRoutingDocrepShard() {
+        ShardRouting primaryRouting = TestShardRouting.newShardRouting(
+            new ShardId(new Index("test_index", "_na_"), 0),
+            "dummy-node",
+            true,
+            ShardRoutingState.STARTED
+        );
+        ShardRouting targetRouting = TestShardRouting.newShardRouting(
+            new ShardId(new Index("test_index", "_na_"), 0),
+            "dummy-node-2",
+            false,
+            ShardRoutingState.STARTED
+        );
+        final ReplicationModeAwareProxy replicationModeAwareProxy = new ReplicationModeAwareProxy(
+            ReplicationMode.NO_REPLICATION,
+            DiscoveryNodes.builder()
+                .add(IndexShardTestUtils.getFakeRemoteEnabledNode(primaryRouting.currentNodeId()))
+                .add(IndexShardTestUtils.getFakeDiscoNode(targetRouting.currentNodeId()))
+                .build(),
+            mock(TransportReplicationAction.ReplicasProxy.class),
+            mock(TransportReplicationAction.ReplicasProxy.class),
+            false
+        );
+        assertEquals(ReplicationMode.FULL_REPLICATION, replicationModeAwareProxy.determineReplicationMode(targetRouting, primaryRouting));
+    }
+
+    /*
+     Replication action originating from remote enabled primary to remote replica during remote store migration
+     Action should not run and proxy should return ReplicationMode.NO_REPLICATION
+     */
+    public void testDetermineReplicationModeTargetRoutingRemoteShard() {
+        ShardRouting primaryRouting = TestShardRouting.newShardRouting(
+            new ShardId(new Index("test_index", "_na_"), 0),
+            "dummy-node",
+            false,
+            ShardRoutingState.STARTED
+        );
+        ShardRouting targetRouting = TestShardRouting.newShardRouting(
+            new ShardId(new Index("test_index", "_na_"), 0),
+            "dummy-node-2",
+            true,
+            ShardRoutingState.STARTED
+        );
+        final ReplicationModeAwareProxy replicationModeAwareProxy = new ReplicationModeAwareProxy(
+            ReplicationMode.NO_REPLICATION,
+            DiscoveryNodes.builder()
+                .add(IndexShardTestUtils.getFakeRemoteEnabledNode(targetRouting.currentNodeId()))
+                .add(IndexShardTestUtils.getFakeRemoteEnabledNode(primaryRouting.currentNodeId()))
+                .build(),
+            mock(TransportReplicationAction.ReplicasProxy.class),
+            mock(TransportReplicationAction.ReplicasProxy.class),
+            false
+        );
+        assertEquals(ReplicationMode.NO_REPLICATION, replicationModeAwareProxy.determineReplicationMode(targetRouting, primaryRouting));
+    }
+
+    /*
+     Replication action originating from remote enabled primary to remote enabled replica during remote store migration
+     with an explicit replication mode specified
+     Action should run and proxy should return the overridden Replication Mode
+     */
+    public void testDetermineReplicationWithExplicitOverrideTargetRoutingRemoteShard() {
+        ReplicationMode replicationModeOverride = ReplicationMode.PRIMARY_TERM_VALIDATION;
+        ShardRouting primaryRouting = TestShardRouting.newShardRouting(
+            new ShardId(new Index("test_index", "_na_"), 0),
+            "dummy-node",
+            false,
+            ShardRoutingState.STARTED
+        );
+        ShardRouting targetRouting = TestShardRouting.newShardRouting(
+            new ShardId(new Index("test_index", "_na_"), 0),
+            "dummy-node-2",
+            true,
+            ShardRoutingState.STARTED
+        );
+        final ReplicationModeAwareProxy replicationModeAwareProxy = new ReplicationModeAwareProxy(
+            replicationModeOverride,
+            DiscoveryNodes.builder()
+                .add(IndexShardTestUtils.getFakeRemoteEnabledNode(targetRouting.currentNodeId()))
+                .add(IndexShardTestUtils.getFakeRemoteEnabledNode(primaryRouting.currentNodeId()))
+                .build(),
+            mock(TransportReplicationAction.ReplicasProxy.class),
+            mock(TransportReplicationAction.ReplicasProxy.class),
+            false
+        );
+        assertEquals(replicationModeOverride, replicationModeAwareProxy.determineReplicationMode(targetRouting, primaryRouting));
+    }
+
+    /*
+     Replication action originating from remote enabled primary with remote enabled index settings enabled
+     Action should not query the DiscoveryNodes object
+     */
+    public void testDetermineReplicationWithRemoteIndexSettingsEnabled() {
+        DiscoveryNodes mockDiscoveryNodes = mock(DiscoveryNodes.class);
+        ShardRouting primaryRouting = TestShardRouting.newShardRouting(
+            new ShardId(new Index("test_index", "_na_"), 0),
+            "dummy-node",
+            false,
+            ShardRoutingState.STARTED
+        );
+        ShardRouting targetRouting = TestShardRouting.newShardRouting(
+            new ShardId(new Index("test_index", "_na_"), 0),
+            "dummy-node-2",
+            true,
+            ShardRoutingState.STARTED
+        );
+        final ReplicationModeAwareProxy replicationModeAwareProxy = new ReplicationModeAwareProxy(
+            ReplicationMode.NO_REPLICATION,
+            mockDiscoveryNodes,
+            mock(TransportReplicationAction.ReplicasProxy.class),
+            mock(TransportReplicationAction.ReplicasProxy.class),
+            true
+        );
+        replicationModeAwareProxy.determineReplicationMode(targetRouting, primaryRouting);
+        // Verify no interactions with the DiscoveryNodes object
+        verify(mockDiscoveryNodes, never()).get(anyString());
+    }
+}
diff --git a/server/src/test/java/org/opensearch/action/support/replication/ReplicationOperationTests.java b/server/src/test/java/org/opensearch/action/support/replication/ReplicationOperationTests.java
index 6b54623b03164..ec5fc1d19e40d 100644
--- a/server/src/test/java/org/opensearch/action/support/replication/ReplicationOperationTests.java
+++ b/server/src/test/java/org/opensearch/action/support/replication/ReplicationOperationTests.java
@@ -48,6 +48,7 @@
 import org.opensearch.cluster.routing.IndexShardRoutingTable;
 import org.opensearch.cluster.routing.ShardRouting;
 import org.opensearch.cluster.routing.ShardRoutingState;
+import org.opensearch.common.collect.Tuple;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.util.concurrent.ConcurrentCollections;
 import org.opensearch.common.util.set.Sets;
@@ -59,6 +60,7 @@
 import org.opensearch.core.index.shard.ShardId;
 import org.opensearch.index.shard.IndexShardNotStartedException;
 import org.opensearch.index.shard.IndexShardState;
+import org.opensearch.index.shard.IndexShardTestUtils;
 import org.opensearch.index.shard.ReplicationGroup;
 import org.opensearch.node.NodeClosedException;
 import org.opensearch.test.OpenSearchTestCase;
@@ -239,7 +241,13 @@ public void testReplicationWithRemoteTranslogEnabled() throws Exception {
             listener,
             replicasProxy,
             0,
-            new ReplicationModeAwareProxy<>(ReplicationMode.NO_REPLICATION, replicasProxy, replicasProxy)
+            new ReplicationModeAwareProxy<>(
+                ReplicationMode.NO_REPLICATION,
+                buildRemoteStoreEnabledDiscoveryNodes(routingTable),
+                replicasProxy,
+                replicasProxy,
+                true
+            )
         );
         op.execute();
         assertTrue("request was not processed on primary", request.processedOnPrimary.get());
@@ -304,7 +312,13 @@ public void testPrimaryToPrimaryReplicationWithRemoteTranslogEnabled() throws Ex
             listener,
             replicasProxy,
             0,
-            new ReplicationModeAwareProxy<>(ReplicationMode.NO_REPLICATION, replicasProxy, replicasProxy)
+            new ReplicationModeAwareProxy<>(
+                ReplicationMode.NO_REPLICATION,
+                buildRemoteStoreEnabledDiscoveryNodes(routingTable),
+                replicasProxy,
+                replicasProxy,
+                true
+            )
         );
         op.execute();
         assertTrue("request was not processed on primary", request.processedOnPrimary.get());
@@ -380,6 +394,144 @@ public void testForceReplicationWithRemoteTranslogEnabled() throws Exception {
         assertEquals(activeIds.size() + initializingIds.size(), shardInfo.getTotal());
     }
 
+    public void testReplicationInDualModeWithDocrepReplica() throws Exception {
+        Set<AllocationId> initializingIds = new HashSet<>();
+        IntStream.range(0, randomIntBetween(2, 5)).forEach(x -> initializingIds.add(AllocationId.newInitializing()));
+        Set<AllocationId> activeIds = new HashSet<>();
+        IntStream.range(0, randomIntBetween(2, 5)).forEach(x -> activeIds.add(AllocationId.newInitializing()));
+
+        AllocationId primaryId = activeIds.iterator().next();
+
+        ShardId shardId = new ShardId("test", "_na_", 0);
+        IndexShardRoutingTable.Builder builder = new IndexShardRoutingTable.Builder(shardId);
+        final ShardRouting primaryShard = newShardRouting(
+            shardId,
+            nodeIdFromAllocationId(primaryId),
+            null,
+            true,
+            ShardRoutingState.STARTED,
+            primaryId
+        );
+        initializingIds.forEach(aId -> {
+            ShardRouting routing = newShardRouting(shardId, nodeIdFromAllocationId(aId), null, false, ShardRoutingState.INITIALIZING, aId);
+            builder.addShard(routing);
+        });
+        activeIds.stream().filter(aId -> !aId.equals(primaryId)).forEach(aId -> {
+            ShardRouting routing = newShardRouting(shardId, nodeIdFromAllocationId(aId), null, false, ShardRoutingState.STARTED, aId);
+            builder.addShard(routing);
+        });
+        builder.addShard(primaryShard);
+        IndexShardRoutingTable routingTable = builder.build();
+
+        Set<String> inSyncAllocationIds = activeIds.stream().map(AllocationId::getId).collect(Collectors.toSet());
+        ReplicationGroup replicationGroup = new ReplicationGroup(routingTable, inSyncAllocationIds, inSyncAllocationIds, 0);
+        List<ShardRouting> replicationTargets = replicationGroup.getReplicationTargets();
+        assertEquals(inSyncAllocationIds.size(), replicationTargets.size());
+        assertTrue(
+            replicationTargets.stream().map(sh -> sh.allocationId().getId()).collect(Collectors.toSet()).containsAll(inSyncAllocationIds)
+        );
+
+        Request request = new Request(shardId);
+        PlainActionFuture<TestPrimary.Result> listener = new PlainActionFuture<>();
+        Map<ShardRouting, Exception> simulatedFailures = new HashMap<>();
+        TestReplicaProxy replicasProxy = new TestReplicaProxy(simulatedFailures);
+        TestPrimary primary = new TestPrimary(primaryShard, () -> replicationGroup, threadPool);
+        final TestReplicationOperation op = new TestReplicationOperation(
+            request,
+            primary,
+            listener,
+            replicasProxy,
+            0,
+            new ReplicationModeAwareProxy<>(
+                ReplicationMode.NO_REPLICATION,
+                buildDiscoveryNodes(routingTable),
+                replicasProxy,
+                replicasProxy,
+                false
+            )
+        );
+        op.execute();
+        assertTrue("request was not processed on primary", request.processedOnPrimary.get());
+        // During dual replication, except for primary, replication action should be executed on all the replicas
+        assertEquals(activeIds.size() - 1, request.processedOnReplicas.size());
+        assertEquals(0, replicasProxy.failedReplicas.size());
+        assertEquals(0, replicasProxy.markedAsStaleCopies.size());
+        assertTrue("post replication operations not run on primary", request.runPostReplicationActionsOnPrimary.get());
+        assertTrue("listener is not marked as done", listener.isDone());
+
+        ShardInfo shardInfo = listener.actionGet().getShardInfo();
+        // All initializing and active shards are set to docrep
+        assertEquals(initializingIds.size() + activeIds.size(), shardInfo.getTotal());
+    }
+
+    public void testReplicationInDualModeWithMixedReplicasSomeInDocrepOthersOnRemote() throws Exception {
+        Set<AllocationId> initializingIds = new HashSet<>();
+        IntStream.range(0, randomIntBetween(2, 5)).forEach(x -> initializingIds.add(AllocationId.newInitializing()));
+        Set<AllocationId> activeIds = new HashSet<>();
+        IntStream.range(0, randomIntBetween(2, 5)).forEach(x -> activeIds.add(AllocationId.newInitializing()));
+
+        AllocationId primaryId = activeIds.iterator().next();
+
+        ShardId shardId = new ShardId("test", "_na_", 0);
+        IndexShardRoutingTable.Builder builder = new IndexShardRoutingTable.Builder(shardId);
+        final ShardRouting primaryShard = newShardRouting(
+            shardId,
+            nodeIdFromAllocationId(primaryId),
+            null,
+            true,
+            ShardRoutingState.STARTED,
+            primaryId
+        );
+        initializingIds.forEach(aId -> {
+            ShardRouting routing = newShardRouting(shardId, nodeIdFromAllocationId(aId), null, false, ShardRoutingState.INITIALIZING, aId);
+            builder.addShard(routing);
+        });
+        activeIds.stream().filter(aId -> !aId.equals(primaryId)).forEach(aId -> {
+            ShardRouting routing = newShardRouting(shardId, nodeIdFromAllocationId(aId), null, false, ShardRoutingState.STARTED, aId);
+            builder.addShard(routing);
+        });
+        builder.addShard(primaryShard);
+        IndexShardRoutingTable routingTable = builder.build();
+
+        Set<String> inSyncAllocationIds = activeIds.stream().map(AllocationId::getId).collect(Collectors.toSet());
+        ReplicationGroup replicationGroup = new ReplicationGroup(routingTable, inSyncAllocationIds, inSyncAllocationIds, 0);
+        List<ShardRouting> replicationTargets = replicationGroup.getReplicationTargets();
+        assertEquals(inSyncAllocationIds.size(), replicationTargets.size());
+        assertTrue(
+            replicationTargets.stream().map(sh -> sh.allocationId().getId()).collect(Collectors.toSet()).containsAll(inSyncAllocationIds)
+        );
+
+        Request request = new Request(shardId);
+        PlainActionFuture<TestPrimary.Result> listener = new PlainActionFuture<>();
+        Map<ShardRouting, Exception> simulatedFailures = new HashMap<>();
+        TestReplicaProxy replicasProxy = new TestReplicaProxy(simulatedFailures);
+        TestPrimary primary = new TestPrimary(primaryShard, () -> replicationGroup, threadPool);
+        // Generating data nodes in mixed mode wherein some of the allocated replicas
+        // are in docrep nodes whereas others are on remote enabled ones
+        Tuple<Integer, DiscoveryNodes> discoveryNodesDetails = buildMixedModeDiscoveryNodes(routingTable);
+        int docRepNodes = discoveryNodesDetails.v1();
+        final TestReplicationOperation op = new TestReplicationOperation(
+            request,
+            primary,
+            listener,
+            replicasProxy,
+            0,
+            new ReplicationModeAwareProxy<>(ReplicationMode.NO_REPLICATION, discoveryNodesDetails.v2(), replicasProxy, replicasProxy, false)
+        );
+        op.execute();
+        assertTrue("request was not processed on primary", request.processedOnPrimary.get());
+        // Only docrep nodes should have the request fanned out to
+        assertEquals(docRepNodes, request.processedOnReplicas.size());
+        assertEquals(0, replicasProxy.failedReplicas.size());
+        assertEquals(0, replicasProxy.markedAsStaleCopies.size());
+        assertTrue("post replication operations not run on primary", request.runPostReplicationActionsOnPrimary.get());
+        assertTrue("listener is not marked as done", listener.isDone());
+
+        ShardInfo shardInfo = listener.actionGet().getShardInfo();
+        // Listener should be invoked for initializing Ids, primary and the operations on docrep nodes
+        assertEquals(1 + docRepNodes + initializingIds.size(), shardInfo.getTotal());
+    }
+
     static String nodeIdFromAllocationId(final AllocationId allocationId) {
         return "n-" + allocationId.getId().substring(0, 8);
     }
@@ -816,6 +968,46 @@ private Set<ShardRouting> getExpectedReplicas(ShardId shardId, ClusterState stat
         return expectedReplicas;
     }
 
+    private DiscoveryNodes buildRemoteStoreEnabledDiscoveryNodes(IndexShardRoutingTable routingTable) {
+        DiscoveryNodes.Builder builder = DiscoveryNodes.builder();
+        for (ShardRouting shardRouting : routingTable) {
+            builder.add(IndexShardTestUtils.getFakeRemoteEnabledNode(shardRouting.currentNodeId()));
+        }
+        return builder.build();
+    }
+
+    private DiscoveryNodes buildDiscoveryNodes(IndexShardRoutingTable routingTable) {
+        DiscoveryNodes.Builder builder = DiscoveryNodes.builder();
+        for (ShardRouting shardRouting : routingTable) {
+            if (shardRouting.primary()) {
+                builder.add(IndexShardTestUtils.getFakeRemoteEnabledNode(shardRouting.currentNodeId()));
+            } else {
+                builder.add(IndexShardTestUtils.getFakeDiscoNode(shardRouting.currentNodeId()));
+            }
+        }
+        return builder.build();
+    }
+
+    private Tuple<Integer, DiscoveryNodes> buildMixedModeDiscoveryNodes(IndexShardRoutingTable routingTable) {
+        int docrepNodes = 0;
+        DiscoveryNodes.Builder builder = DiscoveryNodes.builder();
+        for (ShardRouting shardRouting : routingTable) {
+            if (shardRouting.primary()) {
+                builder.add(IndexShardTestUtils.getFakeRemoteEnabledNode(shardRouting.currentNodeId()));
+            } else {
+                // Only add docrep nodes for allocationIds that are active
+                // since the test cases creates replication group with active allocationIds only
+                if (shardRouting.active() && randomBoolean()) {
+                    builder.add(IndexShardTestUtils.getFakeDiscoNode(shardRouting.currentNodeId()));
+                    docrepNodes += 1;
+                } else {
+                    builder.add(IndexShardTestUtils.getFakeRemoteEnabledNode(shardRouting.currentNodeId()));
+                }
+            }
+        }
+        return new Tuple<>(docrepNodes, builder.build());
+    }
+
     public static class Request extends ReplicationRequest<Request> {
         public AtomicBoolean processedOnPrimary = new AtomicBoolean();
         public AtomicBoolean runPostReplicationActionsOnPrimary = new AtomicBoolean();
diff --git a/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationActionTests.java b/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationActionTests.java
index dad0fa0efd3ec..4a18778cc0b2b 100644
--- a/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationActionTests.java
+++ b/server/src/test/java/org/opensearch/action/support/replication/TransportReplicationActionTests.java
@@ -78,6 +78,7 @@
 import org.opensearch.core.transport.TransportResponse;
 import org.opensearch.index.IndexNotFoundException;
 import org.opensearch.index.IndexService;
+import org.opensearch.index.remote.RemoteStoreTestsHelper;
 import org.opensearch.index.shard.IndexShard;
 import org.opensearch.index.shard.IndexShardClosedException;
 import org.opensearch.index.shard.IndexShardState;
@@ -1589,9 +1590,15 @@ private IndexService mockIndexService(final IndexMetadata indexMetadata, Cluster
 
     @SuppressWarnings("unchecked")
     private IndexShard mockIndexShard(ShardId shardId, ClusterService clusterService) {
+        return mockIndexShard(shardId, clusterService, false);
+    }
+
+    @SuppressWarnings("unchecked")
+    private IndexShard mockIndexShard(ShardId shardId, ClusterService clusterService, boolean remote) {
         final IndexShard indexShard = mock(IndexShard.class);
         when(indexShard.shardId()).thenReturn(shardId);
         when(indexShard.state()).thenReturn(IndexShardState.STARTED);
+        when(indexShard.indexSettings()).thenReturn(RemoteStoreTestsHelper.createIndexSettings(remote));
         doAnswer(invocation -> {
             ActionListener<Releasable> callback = (ActionListener<Releasable>) invocation.getArguments()[0];
             if (isPrimaryMode.get()) {
diff --git a/server/src/test/java/org/opensearch/cluster/ClusterModuleTests.java b/server/src/test/java/org/opensearch/cluster/ClusterModuleTests.java
index 7f53f07bc4788..ae35d37fe77b2 100644
--- a/server/src/test/java/org/opensearch/cluster/ClusterModuleTests.java
+++ b/server/src/test/java/org/opensearch/cluster/ClusterModuleTests.java
@@ -75,6 +75,7 @@
 import org.opensearch.telemetry.metrics.noop.NoopMetricsRegistry;
 import org.opensearch.test.ClusterServiceUtils;
 import org.opensearch.test.gateway.TestGatewayAllocator;
+import org.opensearch.test.gateway.TestShardBatchGatewayAllocator;
 
 import java.util.Arrays;
 import java.util.Collection;
@@ -168,7 +169,7 @@ public void testRegisterAllocationDeciderDuplicate() {
                 public Collection<AllocationDecider> createAllocationDeciders(Settings settings, ClusterSettings clusterSettings) {
                     return Collections.singletonList(new EnableAllocationDecider(settings, clusterSettings));
                 }
-            }), clusterInfoService, null, threadContext, NoopMetricsRegistry.INSTANCE)
+            }), clusterInfoService, null, threadContext, new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE))
         );
         assertEquals(e.getMessage(), "Cannot specify allocation decider [" + EnableAllocationDecider.class.getName() + "] twice");
     }
@@ -179,7 +180,7 @@ public void testRegisterAllocationDecider() {
             public Collection<AllocationDecider> createAllocationDeciders(Settings settings, ClusterSettings clusterSettings) {
                 return Collections.singletonList(new FakeAllocationDecider());
             }
-        }), clusterInfoService, null, threadContext, NoopMetricsRegistry.INSTANCE);
+        }), clusterInfoService, null, threadContext, new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE));
         assertTrue(module.deciderList.stream().anyMatch(d -> d.getClass().equals(FakeAllocationDecider.class)));
     }
 
@@ -189,7 +190,7 @@ private ClusterModule newClusterModuleWithShardsAllocator(Settings settings, Str
             public Map<String, Supplier<ShardsAllocator>> getShardsAllocators(Settings settings, ClusterSettings clusterSettings) {
                 return Collections.singletonMap(name, supplier);
             }
-        }), clusterInfoService, null, threadContext, NoopMetricsRegistry.INSTANCE);
+        }), clusterInfoService, null, threadContext, new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE));
     }
 
     public void testRegisterShardsAllocator() {
@@ -217,7 +218,7 @@ public void testUnknownShardsAllocator() {
                 clusterInfoService,
                 null,
                 threadContext,
-                NoopMetricsRegistry.INSTANCE
+                new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE)
             )
         );
         assertEquals("Unknown ShardsAllocator [dne]", e.getMessage());
@@ -305,9 +306,12 @@ public void testRejectsReservedExistingShardsAllocatorName() {
             clusterInfoService,
             null,
             threadContext,
-            NoopMetricsRegistry.INSTANCE
+            new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE)
+        );
+        expectThrows(
+            IllegalArgumentException.class,
+            () -> clusterModule.setExistingShardsAllocators(new TestGatewayAllocator(), new TestShardBatchGatewayAllocator())
         );
-        expectThrows(IllegalArgumentException.class, () -> clusterModule.setExistingShardsAllocators(new TestGatewayAllocator()));
     }
 
     public void testRejectsDuplicateExistingShardsAllocatorName() {
@@ -318,9 +322,12 @@ public void testRejectsDuplicateExistingShardsAllocatorName() {
             clusterInfoService,
             null,
             threadContext,
-            NoopMetricsRegistry.INSTANCE
+            new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE)
+        );
+        expectThrows(
+            IllegalArgumentException.class,
+            () -> clusterModule.setExistingShardsAllocators(new TestGatewayAllocator(), new TestShardBatchGatewayAllocator())
         );
-        expectThrows(IllegalArgumentException.class, () -> clusterModule.setExistingShardsAllocators(new TestGatewayAllocator()));
     }
 
     private static ClusterPlugin existingShardsAllocatorPlugin(final String allocatorName) {
diff --git a/server/src/test/java/org/opensearch/cluster/DiskUsageTests.java b/server/src/test/java/org/opensearch/cluster/DiskUsageTests.java
index ff47ec3015697..5539dd26dd52d 100644
--- a/server/src/test/java/org/opensearch/cluster/DiskUsageTests.java
+++ b/server/src/test/java/org/opensearch/cluster/DiskUsageTests.java
@@ -194,6 +194,7 @@ public void testFillDiskUsage() {
                 null,
                 null,
                 null,
+                null,
                 null
             ),
             new NodeStats(
@@ -224,6 +225,7 @@ public void testFillDiskUsage() {
                 null,
                 null,
                 null,
+                null,
                 null
             ),
             new NodeStats(
@@ -254,6 +256,7 @@ public void testFillDiskUsage() {
                 null,
                 null,
                 null,
+                null,
                 null
             )
         );
@@ -315,6 +318,7 @@ public void testFillDiskUsageSomeInvalidValues() {
                 null,
                 null,
                 null,
+                null,
                 null
             ),
             new NodeStats(
@@ -345,6 +349,7 @@ public void testFillDiskUsageSomeInvalidValues() {
                 null,
                 null,
                 null,
+                null,
                 null
             ),
             new NodeStats(
@@ -375,6 +380,7 @@ public void testFillDiskUsageSomeInvalidValues() {
                 null,
                 null,
                 null,
+                null,
                 null
             )
         );
diff --git a/server/src/test/java/org/opensearch/cluster/InternalClusterInfoServiceSchedulingTests.java b/server/src/test/java/org/opensearch/cluster/InternalClusterInfoServiceSchedulingTests.java
index 75d5dfc6405ed..537b2d13ec08a 100644
--- a/server/src/test/java/org/opensearch/cluster/InternalClusterInfoServiceSchedulingTests.java
+++ b/server/src/test/java/org/opensearch/cluster/InternalClusterInfoServiceSchedulingTests.java
@@ -89,7 +89,7 @@ public void testScheduling() {
             settings,
             clusterSettings,
             threadPool,
-            NoopMetricsRegistry.INSTANCE
+            new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE)
         ) {
             @Override
             protected PrioritizedOpenSearchThreadPoolExecutor createThreadPoolExecutor() {
diff --git a/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java b/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java
index 5eafe63e63fad..3e343e95f6c4b 100644
--- a/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java
+++ b/server/src/test/java/org/opensearch/cluster/coordination/JoinTaskExecutorTests.java
@@ -76,6 +76,7 @@
 import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY;
 import static org.opensearch.node.remotestore.RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING;
 import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING;
+import static org.opensearch.test.VersionUtils.allOpenSearchVersions;
 import static org.opensearch.test.VersionUtils.allVersions;
 import static org.opensearch.test.VersionUtils.maxCompatibleVersion;
 import static org.opensearch.test.VersionUtils.randomCompatibleVersion;
@@ -885,6 +886,64 @@ public void testUpdatesClusterStateWithMultiNodeClusterAndSameRepository() throw
         validateRepositoryMetadata(result.resultingState, clusterManagerNode, 2);
     }
 
+    public void testNodeJoinInMixedMode() {
+        Settings nodeSettings = Settings.builder().put(REMOTE_STORE_MIGRATION_EXPERIMENTAL, "true").build();
+        FeatureFlags.initializeFeatureFlags(nodeSettings);
+
+        List<Version> versions = allOpenSearchVersions();
+        assert versions.size() >= 2 : "test requires at least two open search versions";
+        Version baseVersion = versions.get(versions.size() - 2);
+        Version higherVersion = versions.get(versions.size() - 1);
+
+        DiscoveryNode currentNode1 = new DiscoveryNode(UUIDs.base64UUID(), buildNewFakeTransportAddress(), baseVersion);
+        DiscoveryNode currentNode2 = new DiscoveryNode(UUIDs.base64UUID(), buildNewFakeTransportAddress(), baseVersion);
+        DiscoveryNodes currentNodes = DiscoveryNodes.builder()
+            .add(currentNode1)
+            .localNodeId(currentNode1.getId())
+            .add(currentNode2)
+            .localNodeId(currentNode2.getId())
+            .build();
+
+        Settings mixedModeCompatibilitySettings = Settings.builder()
+            .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), RemoteStoreNodeService.CompatibilityMode.MIXED)
+            .build();
+
+        Metadata metadata = Metadata.builder().persistentSettings(mixedModeCompatibilitySettings).build();
+
+        // joining node of a higher version than the current nodes
+        DiscoveryNode joiningNode1 = new DiscoveryNode(
+            randomAlphaOfLength(10),
+            randomAlphaOfLength(10),
+            buildNewFakeTransportAddress(),
+            remoteStoreNodeAttributes(SEGMENT_REPO, TRANSLOG_REPO),
+            Collections.singleton(DiscoveryNodeRole.CLUSTER_MANAGER_ROLE),
+            higherVersion
+        );
+        final IllegalStateException exception = expectThrows(
+            IllegalStateException.class,
+            () -> JoinTaskExecutor.ensureNodesCompatibility(joiningNode1, currentNodes, metadata)
+        );
+        String reason = String.format(
+            Locale.ROOT,
+            "remote migration : a node [%s] of higher version [%s] is not allowed to join a cluster with maximum version [%s]",
+            joiningNode1,
+            joiningNode1.getVersion(),
+            currentNodes.getMaxNodeVersion()
+        );
+        assertEquals(reason, exception.getMessage());
+
+        // joining node of the same version as the current nodes
+        DiscoveryNode joiningNode2 = new DiscoveryNode(
+            randomAlphaOfLength(10),
+            randomAlphaOfLength(10),
+            buildNewFakeTransportAddress(),
+            remoteStoreNodeAttributes(SEGMENT_REPO, TRANSLOG_REPO),
+            Collections.singleton(DiscoveryNodeRole.CLUSTER_MANAGER_ROLE),
+            baseVersion
+        );
+        JoinTaskExecutor.ensureNodesCompatibility(joiningNode2, currentNodes, metadata);
+    }
+
     private void validateRepositoryMetadata(ClusterState updatedState, DiscoveryNode existingNode, int expectedRepositories)
         throws Exception {
 
diff --git a/server/src/test/java/org/opensearch/cluster/coordination/NodeJoinTests.java b/server/src/test/java/org/opensearch/cluster/coordination/NodeJoinTests.java
index e9ff8f10799b9..10d5dceb74f55 100644
--- a/server/src/test/java/org/opensearch/cluster/coordination/NodeJoinTests.java
+++ b/server/src/test/java/org/opensearch/cluster/coordination/NodeJoinTests.java
@@ -33,6 +33,7 @@
 
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.opensearch.Version;
+import org.opensearch.cluster.ClusterManagerMetrics;
 import org.opensearch.cluster.ClusterName;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.OpenSearchAllocationTestCase;
@@ -181,7 +182,7 @@ private void setupRealClusterManagerServiceAndCoordinator(long term, ClusterStat
             Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), "test_node").build(),
             new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
             threadPool,
-            NoopMetricsRegistry.INSTANCE
+            new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE)
         );
         AtomicReference<ClusterState> clusterStateRef = new AtomicReference<>(initialState);
         clusterManagerService.setClusterStatePublisher((event, publishListener, ackListener) -> {
diff --git a/server/src/test/java/org/opensearch/cluster/metadata/IndexNameExpressionResolverTests.java b/server/src/test/java/org/opensearch/cluster/metadata/IndexNameExpressionResolverTests.java
index 51696ca599d02..fda2f411b1994 100644
--- a/server/src/test/java/org/opensearch/cluster/metadata/IndexNameExpressionResolverTests.java
+++ b/server/src/test/java/org/opensearch/cluster/metadata/IndexNameExpressionResolverTests.java
@@ -352,6 +352,25 @@ public void testIndexOptionsWildcardExpansion() {
         assertEquals(1, results.length);
         assertThat(results, arrayContainingInAnyOrder(".hidden-closed"));
 
+        options = IndicesOptions.fromOptions(false, true, false, true, false, false, false, false);
+        context = new IndexNameExpressionResolver.Context(state, options, false);
+
+        results = indexNameExpressionResolver.concreteIndexNames(context, "foo*");
+        assertEquals(1, results.length);
+        assertEquals("foo", results[0]);
+
+        try {
+            options = IndicesOptions.fromOptions(false, true, false, true, false, true, false, false);
+            context = new IndexNameExpressionResolver.Context(state, options, false);
+
+            results = indexNameExpressionResolver.concreteIndexNames(context, "foo*");
+            assertEquals(1, results.length);
+            assertEquals("foo", results[0]);
+        } catch (IllegalArgumentException iae) {
+            String expectedMessage = "To expand [CLOSE] wildcard, please set forbid_closed_indices to `false`";
+            assertEquals(expectedMessage, iae.getMessage());
+        }
+
         // Only open
         options = IndicesOptions.fromOptions(false, true, true, false, false);
         context = new IndexNameExpressionResolver.Context(state, options, false);
diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java
index cf4de32890a2a..fad98a6609c3b 100644
--- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java
+++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java
@@ -55,6 +55,7 @@
 import org.opensearch.cluster.routing.allocation.decider.AwarenessAllocationDecider;
 import org.opensearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider;
 import org.opensearch.cluster.service.ClusterService;
+import org.opensearch.common.UUIDs;
 import org.opensearch.common.compress.CompressedXContent;
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.IndexScopedSettings;
@@ -62,6 +63,7 @@
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.util.BigArrays;
+import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.common.xcontent.XContentFactory;
 import org.opensearch.core.index.Index;
 import org.opensearch.core.xcontent.NamedXContentRegistry;
@@ -71,16 +73,20 @@
 import org.opensearch.index.IndexSettings;
 import org.opensearch.index.mapper.MapperService;
 import org.opensearch.index.query.QueryShardContext;
-import org.opensearch.index.remote.RemoteStorePathType;
+import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm;
+import org.opensearch.index.remote.RemoteStoreEnums.PathType;
 import org.opensearch.index.translog.Translog;
+import org.opensearch.indices.DefaultRemoteStoreSettings;
 import org.opensearch.indices.IndexCreationException;
 import org.opensearch.indices.IndicesService;
 import org.opensearch.indices.InvalidAliasNameException;
 import org.opensearch.indices.InvalidIndexNameException;
+import org.opensearch.indices.RemoteStoreSettings;
 import org.opensearch.indices.ShardLimitValidator;
 import org.opensearch.indices.SystemIndexDescriptor;
 import org.opensearch.indices.SystemIndices;
 import org.opensearch.indices.replication.common.ReplicationType;
+import org.opensearch.node.remotestore.RemoteStoreNodeService;
 import org.opensearch.snapshots.EmptySnapshotsInfoService;
 import org.opensearch.test.ClusterServiceUtils;
 import org.opensearch.test.OpenSearchTestCase;
@@ -135,6 +141,7 @@
 import static org.opensearch.cluster.metadata.MetadataCreateIndexService.getIndexNumberOfRoutingShards;
 import static org.opensearch.cluster.metadata.MetadataCreateIndexService.parseV1Mappings;
 import static org.opensearch.cluster.metadata.MetadataCreateIndexService.resolveAndValidateAliases;
+import static org.opensearch.common.util.FeatureFlags.REMOTE_STORE_MIGRATION_EXPERIMENTAL;
 import static org.opensearch.index.IndexModule.INDEX_STORE_TYPE_SETTING;
 import static org.opensearch.index.IndexSettings.INDEX_REFRESH_INTERVAL_SETTING;
 import static org.opensearch.index.IndexSettings.INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING;
@@ -149,6 +156,8 @@
 import static org.opensearch.node.Node.NODE_ATTRIBUTES;
 import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY;
 import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY;
+import static org.opensearch.node.remotestore.RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING;
+import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.endsWith;
 import static org.hamcrest.Matchers.equalTo;
@@ -695,7 +704,8 @@ public void testValidateIndexName() throws Exception {
                 null,
                 new SystemIndices(Collections.emptyMap()),
                 false,
-                new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings())
+                new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings()),
+                DefaultRemoteStoreSettings.INSTANCE
             );
             validateIndexName(
                 checkerService,
@@ -781,7 +791,8 @@ public void testValidateDotIndex() {
                 null,
                 new SystemIndices(Collections.singletonMap("foo", systemIndexDescriptors)),
                 false,
-                new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings())
+                new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings()),
+                DefaultRemoteStoreSettings.INSTANCE
             );
             // Check deprecations
             assertFalse(checkerService.validateDotIndex(".test2", false));
@@ -1206,7 +1217,8 @@ public void testvalidateIndexSettings() {
             null,
             new SystemIndices(Collections.emptyMap()),
             true,
-            new AwarenessReplicaBalance(settings, clusterService.getClusterSettings())
+            new AwarenessReplicaBalance(settings, clusterService.getClusterSettings()),
+            DefaultRemoteStoreSettings.INSTANCE
         );
 
         List<String> validationErrors = checkerService.getIndexSettingsValidationErrors(settings, false, Optional.empty());
@@ -1325,7 +1337,8 @@ public void testClusterForceReplicationTypeInValidateIndexSettings() {
             null,
             new SystemIndices(Collections.emptyMap()),
             true,
-            new AwarenessReplicaBalance(forceClusterSettingEnabled, clusterService.getClusterSettings())
+            new AwarenessReplicaBalance(forceClusterSettingEnabled, clusterService.getClusterSettings()),
+            DefaultRemoteStoreSettings.INSTANCE
         );
         // Use DOCUMENT replication type setting for index creation
         final Settings indexSettings = Settings.builder().put(INDEX_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.DOCUMENT).build();
@@ -1348,18 +1361,16 @@ public void testClusterForceReplicationTypeInValidateIndexSettings() {
     }
 
     public void testRemoteStoreNoUserOverrideExceptReplicationTypeSegmentIndexSettings() {
-        Settings settings = Settings.builder()
-            .put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.DOCUMENT)
-            .put(segmentRepositoryNameAttributeKey, "my-segment-repo-1")
-            .put(translogRepositoryNameAttributeKey, "my-translog-repo-1")
+        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
+            .nodes(DiscoveryNodes.builder().add(getRemoteNode()).build())
             .build();
-
+        Settings settings = Settings.builder().put(translogRepositoryNameAttributeKey, "my-translog-repo-1").build();
         request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test");
         final Settings.Builder requestSettings = Settings.builder();
         requestSettings.put(SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT);
         request.settings(requestSettings.build());
         Settings indexSettings = aggregateIndexSettings(
-            ClusterState.EMPTY_STATE,
+            clusterState,
             request,
             Settings.EMPTY,
             null,
@@ -1375,21 +1386,20 @@ public void testRemoteStoreNoUserOverrideExceptReplicationTypeSegmentIndexSettin
             "my-segment-repo-1",
             "my-translog-repo-1",
             ReplicationType.SEGMENT.toString(),
-            IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL
+            null
         );
     }
 
     public void testRemoteStoreImplicitOverrideReplicationTypeToSegmentForRemoteStore() {
-        Settings settings = Settings.builder()
-            .put(segmentRepositoryNameAttributeKey, "my-segment-repo-1")
-            .put(translogRepositoryNameAttributeKey, "my-translog-repo-1")
+        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
+            .nodes(DiscoveryNodes.builder().add(getRemoteNode()).build())
             .build();
-
+        Settings settings = Settings.builder().put(translogRepositoryNameAttributeKey, "my-translog-repo-1").build();
         request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test");
         final Settings.Builder requestSettings = Settings.builder();
         request.settings(requestSettings.build());
         Settings indexSettings = aggregateIndexSettings(
-            ClusterState.EMPTY_STATE,
+            clusterState,
             request,
             Settings.EMPTY,
             null,
@@ -1405,20 +1415,18 @@ public void testRemoteStoreImplicitOverrideReplicationTypeToSegmentForRemoteStor
             "my-segment-repo-1",
             "my-translog-repo-1",
             ReplicationType.SEGMENT.toString(),
-            IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL
+            null
         );
     }
 
     public void testRemoteStoreNoUserOverrideIndexSettings() {
-        Settings settings = Settings.builder()
-            .put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), ReplicationType.SEGMENT)
-            .put(segmentRepositoryNameAttributeKey, "my-segment-repo-1")
-            .put(translogRepositoryNameAttributeKey, "my-translog-repo-1")
+        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT)
+            .nodes(DiscoveryNodes.builder().add(getRemoteNode()).build())
             .build();
-
+        Settings settings = Settings.builder().put(translogRepositoryNameAttributeKey, "my-translog-repo-1").build();
         request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test");
         Settings indexSettings = aggregateIndexSettings(
-            ClusterState.EMPTY_STATE,
+            clusterState,
             request,
             Settings.EMPTY,
             null,
@@ -1434,7 +1442,7 @@ public void testRemoteStoreNoUserOverrideIndexSettings() {
             "my-segment-repo-1",
             "my-translog-repo-1",
             ReplicationType.SEGMENT.toString(),
-            IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL
+            null
         );
     }
 
@@ -1455,7 +1463,8 @@ public void testRemoteStoreDisabledByUserIndexSettings() {
                 null,
                 new SystemIndices(Collections.emptyMap()),
                 true,
-                new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings())
+                new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings()),
+                DefaultRemoteStoreSettings.INSTANCE
             );
 
             final List<String> validationErrors = checkerService.getIndexSettingsValidationErrors(
@@ -1489,7 +1498,8 @@ public void testRemoteStoreOverrideSegmentRepoIndexSettings() {
                 null,
                 new SystemIndices(Collections.emptyMap()),
                 true,
-                new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings())
+                new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings()),
+                DefaultRemoteStoreSettings.INSTANCE
             );
 
             final List<String> validationErrors = checkerService.getIndexSettingsValidationErrors(
@@ -1528,7 +1538,8 @@ public void testRemoteStoreOverrideTranslogRepoIndexSettings() {
                 null,
                 new SystemIndices(Collections.emptyMap()),
                 true,
-                new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings())
+                new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings()),
+                DefaultRemoteStoreSettings.INSTANCE
             );
 
             final List<String> validationErrors = checkerService.getIndexSettingsValidationErrors(
@@ -1550,6 +1561,109 @@ public void testRemoteStoreOverrideTranslogRepoIndexSettings() {
         }));
     }
 
+    public void testNewIndexIsRemoteStoreBackedForRemoteStoreDirectionAndMixedMode() {
+        FeatureFlags.initializeFeatureFlags(Settings.builder().put(REMOTE_STORE_MIGRATION_EXPERIMENTAL, "true").build());
+
+        // non-remote cluster manager node
+        DiscoveryNode nonRemoteClusterManagerNode = new DiscoveryNode(UUIDs.base64UUID(), buildNewFakeTransportAddress(), Version.CURRENT);
+
+        DiscoveryNodes discoveryNodes = DiscoveryNodes.builder()
+            .add(nonRemoteClusterManagerNode)
+            .localNodeId(nonRemoteClusterManagerNode.getId())
+            .build();
+
+        ClusterState clusterState = ClusterState.builder(ClusterName.DEFAULT).nodes(discoveryNodes).build();
+
+        ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+
+        request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test");
+
+        Settings indexSettings = aggregateIndexSettings(
+            clusterState,
+            request,
+            Settings.EMPTY,
+            null,
+            Settings.EMPTY,
+            IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
+            randomShardLimitService(),
+            Collections.emptySet(),
+            clusterSettings
+        );
+        verifyRemoteStoreIndexSettings(indexSettings, null, null, null, ReplicationType.DOCUMENT.toString(), null);
+
+        // remote data node
+        DiscoveryNode remoteDataNode = getRemoteNode();
+
+        discoveryNodes = DiscoveryNodes.builder(discoveryNodes).add(remoteDataNode).localNodeId(remoteDataNode.getId()).build();
+
+        clusterState = ClusterState.builder(ClusterName.DEFAULT).nodes(discoveryNodes).build();
+
+        Settings remoteStoreMigrationSettings = Settings.builder()
+            .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), RemoteStoreNodeService.CompatibilityMode.MIXED)
+            .put(MIGRATION_DIRECTION_SETTING.getKey(), RemoteStoreNodeService.Direction.REMOTE_STORE)
+            .build();
+
+        clusterSettings = new ClusterSettings(remoteStoreMigrationSettings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+
+        indexSettings = aggregateIndexSettings(
+            clusterState,
+            request,
+            Settings.EMPTY,
+            null,
+            Settings.EMPTY,
+            IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
+            randomShardLimitService(),
+            Collections.emptySet(),
+            clusterSettings
+        );
+
+        verifyRemoteStoreIndexSettings(
+            indexSettings,
+            "true",
+            "my-segment-repo-1",
+            "my-translog-repo-1",
+            ReplicationType.SEGMENT.toString(),
+            null
+        );
+
+        Map<String, String> missingTranslogAttribute = Map.of(REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, "my-segment-repo-1");
+
+        DiscoveryNodes finalDiscoveryNodes = DiscoveryNodes.builder()
+            .add(nonRemoteClusterManagerNode)
+            .add(
+                new DiscoveryNode(
+                    UUIDs.base64UUID(),
+                    buildNewFakeTransportAddress(),
+                    missingTranslogAttribute,
+                    Set.of(DiscoveryNodeRole.INGEST_ROLE, DiscoveryNodeRole.REMOTE_CLUSTER_CLIENT_ROLE, DiscoveryNodeRole.DATA_ROLE),
+                    Version.CURRENT
+                )
+            )
+            .build();
+
+        ClusterState finalClusterState = ClusterState.builder(ClusterName.DEFAULT).nodes(finalDiscoveryNodes).build();
+        ClusterSettings finalClusterSettings = clusterSettings;
+
+        final IndexCreationException error = expectThrows(IndexCreationException.class, () -> {
+            aggregateIndexSettings(
+                finalClusterState,
+                request,
+                Settings.EMPTY,
+                null,
+                Settings.EMPTY,
+                IndexScopedSettings.DEFAULT_SCOPED_SETTINGS,
+                randomShardLimitService(),
+                Collections.emptySet(),
+                finalClusterSettings
+            );
+        });
+
+        assertThat(
+            error.getCause().getMessage(),
+            containsString("Cluster is migrating to remote store but no remote node found, failing index creation")
+        );
+    }
+
     public void testBuildIndexMetadata() {
         IndexMetadata sourceIndexMetadata = IndexMetadata.builder("parent")
             .settings(Settings.builder().put("index.version.created", Version.CURRENT).build())
@@ -1587,32 +1701,35 @@ public void testBuildIndexMetadata() {
      */
     public void testRemoteCustomData() {
         // Case 1 - Remote store is not enabled
-        IndexMetadata indexMetadata = testRemoteCustomData(false, randomFrom(RemoteStorePathType.values()));
+        IndexMetadata indexMetadata = testRemoteCustomData(false, randomFrom(PathType.values()));
         assertNull(indexMetadata.getCustomData(IndexMetadata.REMOTE_STORE_CUSTOM_KEY));
 
         // Case 2 - cluster.remote_store.index.path.prefix.optimised=fixed (default value)
-        indexMetadata = testRemoteCustomData(true, RemoteStorePathType.FIXED);
+        indexMetadata = testRemoteCustomData(true, PathType.FIXED);
+        Map<String, String> remoteCustomData = indexMetadata.getCustomData(IndexMetadata.REMOTE_STORE_CUSTOM_KEY);
+        validateRemoteCustomData(remoteCustomData, PathType.NAME, PathType.FIXED.name());
+        assertNull(remoteCustomData.get(PathHashAlgorithm.NAME));
+
+        // Case 3 - cluster.remote_store.index.path.prefix.optimised=hashed_prefix
+        indexMetadata = testRemoteCustomData(true, PathType.HASHED_PREFIX);
         validateRemoteCustomData(
             indexMetadata.getCustomData(IndexMetadata.REMOTE_STORE_CUSTOM_KEY),
-            RemoteStorePathType.NAME,
-            RemoteStorePathType.FIXED.toString()
+            PathType.NAME,
+            PathType.HASHED_PREFIX.toString()
         );
-
-        // Case 3 - cluster.remote_store.index.path.prefix.optimised=hashed_prefix
-        indexMetadata = testRemoteCustomData(true, RemoteStorePathType.HASHED_PREFIX);
         validateRemoteCustomData(
             indexMetadata.getCustomData(IndexMetadata.REMOTE_STORE_CUSTOM_KEY),
-            RemoteStorePathType.NAME,
-            RemoteStorePathType.HASHED_PREFIX.toString()
+            PathHashAlgorithm.NAME,
+            PathHashAlgorithm.FNV_1A_COMPOSITE_1.name()
         );
     }
 
-    private IndexMetadata testRemoteCustomData(boolean remoteStoreEnabled, RemoteStorePathType remoteStorePathType) {
+    private IndexMetadata testRemoteCustomData(boolean remoteStoreEnabled, PathType pathType) {
         Settings.Builder settingsBuilder = Settings.builder();
         if (remoteStoreEnabled) {
             settingsBuilder.put(NODE_ATTRIBUTES.getKey() + REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, "test");
         }
-        settingsBuilder.put(IndicesService.CLUSTER_REMOTE_STORE_PATH_PREFIX_TYPE_SETTING.getKey(), remoteStorePathType.toString());
+        settingsBuilder.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), pathType.toString());
         Settings settings = settingsBuilder.build();
 
         ClusterService clusterService = mock(ClusterService.class);
@@ -1626,6 +1743,7 @@ private IndexMetadata testRemoteCustomData(boolean remoteStoreEnabled, RemoteSto
         when(clusterService.getSettings()).thenReturn(settings);
         when(clusterService.getClusterSettings()).thenReturn(clusterSettings);
         when(clusterService.state()).thenReturn(clusterState);
+        RemoteStoreSettings remoteStoreSettings = new RemoteStoreSettings(settings, clusterSettings);
 
         ThreadPool threadPool = new TestThreadPool(getTestName());
         MetadataCreateIndexService metadataCreateIndexService = new MetadataCreateIndexService(
@@ -1641,7 +1759,8 @@ private IndexMetadata testRemoteCustomData(boolean remoteStoreEnabled, RemoteSto
             null,
             new SystemIndices(Collections.emptyMap()),
             true,
-            new AwarenessReplicaBalance(settings, clusterService.getClusterSettings())
+            new AwarenessReplicaBalance(settings, clusterService.getClusterSettings()),
+            remoteStoreSettings
         );
         CreateIndexClusterStateUpdateRequest request = new CreateIndexClusterStateUpdateRequest("create index", "test", "test");
         Settings indexSettings = Settings.builder()
@@ -1764,7 +1883,8 @@ public void testIndexLifecycleNameSetting() {
                 null,
                 new SystemIndices(Collections.emptyMap()),
                 true,
-                new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings())
+                new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings()),
+                DefaultRemoteStoreSettings.INSTANCE
             );
 
             final List<String> validationErrors = checkerService.getIndexSettingsValidationErrors(ilnSetting, true, Optional.empty());
@@ -2114,7 +2234,20 @@ private void verifyRemoteStoreIndexSettings(
         assertEquals(isRemoteSegmentEnabled, indexSettings.get(SETTING_REMOTE_STORE_ENABLED));
         assertEquals(remoteSegmentRepo, indexSettings.get(SETTING_REMOTE_SEGMENT_STORE_REPOSITORY));
         assertEquals(remoteTranslogRepo, indexSettings.get(SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY));
-        assertEquals(translogBufferInterval, INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.get(indexSettings));
+        assertEquals(translogBufferInterval, indexSettings.get(INDEX_REMOTE_TRANSLOG_BUFFER_INTERVAL_SETTING.getKey()));
+    }
+
+    private DiscoveryNode getRemoteNode() {
+        Map<String, String> attributes = new HashMap<>();
+        attributes.put(REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, "my-segment-repo-1");
+        attributes.put(REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, "my-translog-repo-1");
+        return new DiscoveryNode(
+            UUIDs.base64UUID(),
+            buildNewFakeTransportAddress(),
+            attributes,
+            DiscoveryNodeRole.BUILT_IN_ROLES,
+            Version.CURRENT
+        );
     }
 
     @After
diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataIndexTemplateServiceTests.java
index 0b8e64e31a523..0b99ffac67ee8 100644
--- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataIndexTemplateServiceTests.java
+++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataIndexTemplateServiceTests.java
@@ -55,6 +55,7 @@
 import org.opensearch.env.Environment;
 import org.opensearch.index.mapper.MapperParsingException;
 import org.opensearch.index.mapper.MapperService;
+import org.opensearch.indices.DefaultRemoteStoreSettings;
 import org.opensearch.indices.IndexTemplateMissingException;
 import org.opensearch.indices.InvalidIndexTemplateException;
 import org.opensearch.indices.SystemIndices;
@@ -2051,7 +2052,8 @@ private static List<Throwable> putTemplate(NamedXContentRegistry xContentRegistr
             xContentRegistry,
             new SystemIndices(Collections.emptyMap()),
             true,
-            new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings())
+            new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings()),
+            DefaultRemoteStoreSettings.INSTANCE
         );
         MetadataIndexTemplateService service = new MetadataIndexTemplateService(
             clusterService,
diff --git a/server/src/test/java/org/opensearch/cluster/routing/IndexShardRoutingTableTests.java b/server/src/test/java/org/opensearch/cluster/routing/IndexShardRoutingTableTests.java
index ebb7529d3f733..e881016fb9305 100644
--- a/server/src/test/java/org/opensearch/cluster/routing/IndexShardRoutingTableTests.java
+++ b/server/src/test/java/org/opensearch/cluster/routing/IndexShardRoutingTableTests.java
@@ -39,6 +39,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.UUID;
 
 public class IndexShardRoutingTableTests extends OpenSearchTestCase {
     public void testEqualsAttributesKey() {
@@ -69,4 +70,48 @@ public void testEquals() {
         assertNotEquals(table1, s);
         assertNotEquals(table1, table3);
     }
+
+    public void testShardsMatchingPredicate() {
+        ShardId shardId = new ShardId(new Index("a", UUID.randomUUID().toString()), 0);
+        ShardRouting primary = TestShardRouting.newShardRouting(shardId, "node-1", true, ShardRoutingState.STARTED);
+        ShardRouting replica = TestShardRouting.newShardRouting(shardId, "node-2", false, ShardRoutingState.STARTED);
+        ShardRouting unassignedReplica = ShardRouting.newUnassigned(
+            shardId,
+            false,
+            RecoverySource.PeerRecoverySource.INSTANCE,
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)
+        );
+        ShardRouting relocatingReplica1 = TestShardRouting.newShardRouting(
+            shardId,
+            "node-3",
+            "node-4",
+            false,
+            ShardRoutingState.RELOCATING
+        );
+        ShardRouting relocatingReplica2 = TestShardRouting.newShardRouting(
+            shardId,
+            "node-4",
+            "node-5",
+            false,
+            ShardRoutingState.RELOCATING
+        );
+
+        IndexShardRoutingTable table = new IndexShardRoutingTable(
+            shardId,
+            Arrays.asList(primary, replica, unassignedReplica, relocatingReplica1, relocatingReplica2)
+        );
+        assertEquals(List.of(primary), table.shardsMatchingPredicate(ShardRouting::primary));
+        assertEquals(
+            List.of(replica, unassignedReplica, relocatingReplica1, relocatingReplica2),
+            table.shardsMatchingPredicate(shardRouting -> !shardRouting.primary())
+        );
+        assertEquals(
+            List.of(unassignedReplica),
+            table.shardsMatchingPredicate(shardRouting -> !shardRouting.primary() && shardRouting.unassigned())
+        );
+        assertEquals(
+            Arrays.asList(relocatingReplica1, relocatingReplica2),
+            table.shardsMatchingPredicate(shardRouting -> !shardRouting.primary() && shardRouting.relocating())
+        );
+    }
 }
diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/AllocationServiceTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/AllocationServiceTests.java
index a82b960473407..cce75105dd33f 100644
--- a/server/src/test/java/org/opensearch/cluster/routing/allocation/AllocationServiceTests.java
+++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/AllocationServiceTests.java
@@ -33,6 +33,7 @@
 
 import org.opensearch.Version;
 import org.opensearch.cluster.ClusterInfo;
+import org.opensearch.cluster.ClusterManagerMetrics;
 import org.opensearch.cluster.ClusterName;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.EmptyClusterInfoService;
@@ -148,7 +149,14 @@ public void testAssignsPrimariesInPriorityOrderThenReplicas() {
         final ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
         final MetricsRegistry metricsRegistry = mock(MetricsRegistry.class);
         final Histogram rerouteHistogram = mock(Histogram.class);
-        when(metricsRegistry.createHistogram(anyString(), anyString(), anyString())).thenReturn(rerouteHistogram);
+        final Histogram mockedHistogram = mock(Histogram.class);
+        when(metricsRegistry.createHistogram(anyString(), anyString(), anyString())).thenAnswer(invocationOnMock -> {
+            String histogramName = (String) invocationOnMock.getArguments()[0];
+            if (histogramName.contains("reroute.latency")) {
+                return rerouteHistogram;
+            }
+            return mockedHistogram;
+        });
         final AllocationService allocationService = new AllocationService(
             new AllocationDeciders(
                 Arrays.asList(
@@ -171,7 +179,7 @@ public ShardAllocationDecision decideShardAllocation(ShardRouting shard, Routing
             },
             new EmptyClusterInfoService(),
             EmptySnapshotsInfoService.INSTANCE,
-            metricsRegistry
+            new ClusterManagerMetrics(metricsRegistry)
         );
 
         final String unrealisticAllocatorName = "unrealistic";
@@ -272,12 +280,17 @@ public ShardAllocationDecision decideShardAllocation(ShardRouting shard, Routing
         assertTrue(routingTable3.index("lowPriority").allPrimaryShardsActive());
         assertThat(routingTable3.index("invalid").shardsWithState(ShardRoutingState.STARTED), empty());
 
-        verify(metricsRegistry, times(1)).createHistogram(anyString(), anyString(), anyString());
         verify(rerouteHistogram, times(3)).record(anyDouble());
     }
 
     public void testExplainsNonAllocationOfShardWithUnknownAllocator() {
-        final AllocationService allocationService = new AllocationService(null, null, null, null, NoopMetricsRegistry.INSTANCE);
+        final AllocationService allocationService = new AllocationService(
+            null,
+            null,
+            null,
+            null,
+            new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE)
+        );
         allocationService.setExistingShardsAllocators(
             Collections.singletonMap(GatewayAllocator.ALLOCATOR_NAME, new TestGatewayAllocator())
         );
diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/BalanceConfigurationTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/BalanceConfigurationTests.java
index 62dce9c4edeb5..11cbe89645657 100644
--- a/server/src/test/java/org/opensearch/cluster/routing/allocation/BalanceConfigurationTests.java
+++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/BalanceConfigurationTests.java
@@ -72,6 +72,7 @@
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
+import static org.opensearch.cluster.ClusterName.CLUSTER_NAME_SETTING;
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_CREATION_DATE;
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS;
@@ -140,10 +141,14 @@ public void testIndexBalance() {
     }
 
     private Settings.Builder getSettingsBuilderForPrimaryBalance() {
-        return getSettingsBuilderForPrimaryBalance(true);
+        return getSettingsBuilderForPrimaryBalance(true, false);
     }
 
-    private Settings.Builder getSettingsBuilderForPrimaryBalance(boolean preferPrimaryBalance) {
+    private Settings.Builder getSettingsBuilderForPrimaryReBalance() {
+        return getSettingsBuilderForPrimaryBalance(true, true);
+    }
+
+    private Settings.Builder getSettingsBuilderForPrimaryBalance(boolean preferPrimaryBalance, boolean preferPrimaryRebalance) {
         final float indexBalance = 0.55f;
         final float shardBalance = 0.45f;
         final float balanceThreshold = 1.0f;
@@ -155,6 +160,7 @@ private Settings.Builder getSettingsBuilderForPrimaryBalance(boolean preferPrima
         );
         settings.put(BalancedShardsAllocator.INDEX_BALANCE_FACTOR_SETTING.getKey(), indexBalance);
         settings.put(BalancedShardsAllocator.PREFER_PRIMARY_SHARD_BALANCE.getKey(), preferPrimaryBalance);
+        settings.put(BalancedShardsAllocator.PREFER_PRIMARY_SHARD_REBALANCE.getKey(), preferPrimaryRebalance);
         settings.put(BalancedShardsAllocator.SHARD_BALANCE_FACTOR_SETTING.getKey(), shardBalance);
         settings.put(BalancedShardsAllocator.THRESHOLD_SETTING.getKey(), balanceThreshold);
         return settings;
@@ -201,7 +207,7 @@ public void testPrimaryBalanceWithoutPreferPrimaryBalanceSetting() {
         int balanceFailed = 0;
 
         AllocationService strategy = createAllocationService(
-            getSettingsBuilderForPrimaryBalance(false).build(),
+            getSettingsBuilderForPrimaryBalance(false, false).build(),
             new TestGatewayAllocator()
         );
         for (int i = 0; i < numberOfRuns; i++) {
@@ -244,6 +250,60 @@ public void testPrimaryBalanceWithPreferPrimaryBalanceSetting() {
         assertTrue(balanceFailed <= 1);
     }
 
+    /**
+     * This test verifies primary shard balance is attained  setting.
+     */
+    public void testPrimaryBalanceNotSolvedForNodeDropWithPreferPrimaryBalanceSetting() {
+        final int numberOfNodes = 4;
+        final int numberOfIndices = 4;
+        final int numberOfShards = 4;
+        final int numberOfReplicas = 1;
+        final int numberOfRuns = 5;
+        final float buffer = 0.10f;
+        int balanceFailed = 0;
+
+        AllocationService strategy = createAllocationService(getSettingsBuilderForPrimaryBalance().build(), new TestGatewayAllocator());
+        for (int i = 0; i < numberOfRuns; i++) {
+            ClusterState clusterState = initCluster(strategy, numberOfIndices, numberOfNodes, numberOfShards, numberOfReplicas);
+            clusterState = removeOneNode(clusterState, strategy);
+            logger.info(ShardAllocations.printShardDistribution(clusterState));
+            try {
+                verifyPrimaryBalance(clusterState, buffer);
+            } catch (AssertionError | Exception e) {
+                balanceFailed++;
+                logger.info("Unexpected assertion failure");
+            }
+        }
+        assertTrue(balanceFailed >= 4);
+    }
+
+    /**
+     * This test verifies primary shard balance is attained with PREFER_PRIMARY_SHARD_BALANCE setting.
+     */
+    public void testPrimaryBalanceSolvedWithPreferPrimaryRebalanceSetting() {
+        final int numberOfNodes = 4;
+        final int numberOfIndices = 4;
+        final int numberOfShards = 4;
+        final int numberOfReplicas = 1;
+        final int numberOfRuns = 5;
+        final float buffer = 0.10f;
+        int balanceFailed = 0;
+
+        AllocationService strategy = createAllocationService(getSettingsBuilderForPrimaryReBalance().build(), new TestGatewayAllocator());
+        for (int i = 0; i < numberOfRuns; i++) {
+            ClusterState clusterState = initCluster(strategy, numberOfIndices, numberOfNodes, numberOfShards, numberOfReplicas);
+            clusterState = removeOneNode(clusterState, strategy);
+            logger.info(ShardAllocations.printShardDistribution(clusterState));
+            try {
+                verifyPrimaryBalance(clusterState, buffer);
+            } catch (Exception e) {
+                balanceFailed++;
+                logger.info("Unexpected assertion failure");
+            }
+        }
+        assertTrue(balanceFailed <= 1);
+    }
+
     /**
      * This test verifies the allocation logic when nodes breach multiple constraints and ensure node breaching min
      * constraints chosen for allocation.
@@ -367,8 +427,7 @@ public void testPrimaryBalanceWithContrainstBreaching() {
      */
     public void testGlobalPrimaryBalance() throws Exception {
         AllocationService strategy = createAllocationService(getSettingsBuilderForPrimaryBalance().build(), new TestGatewayAllocator());
-        ClusterState clusterState = ClusterState.builder(org.opensearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
-            .build();
+        ClusterState clusterState = ClusterState.builder(CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).build();
         clusterState = addNode(clusterState, strategy);
         clusterState = addNode(clusterState, strategy);
 
@@ -377,7 +436,30 @@ public void testGlobalPrimaryBalance() throws Exception {
         clusterState = addIndex(clusterState, strategy, "test-index3", 1, 1);
 
         logger.info(ShardAllocations.printShardDistribution(clusterState));
-        verifyPrimaryBalance(clusterState);
+        verifyPrimaryBalance(clusterState, 0.0f);
+    }
+
+    /**
+     * This test verifies global balance by creating indices iteratively and verify primary shards do not pile up on one
+     * @throws Exception generic exception
+     */
+    public void testGlobalPrimaryBalanceWithNodeDrops() throws Exception {
+        final float buffer = 0.10f;
+        AllocationService strategy = createAllocationService(getSettingsBuilderForPrimaryReBalance().build(), new TestGatewayAllocator());
+        ClusterState clusterState = ClusterState.builder(CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY)).build();
+        clusterState = addNodes(clusterState, strategy, 5);
+
+        clusterState = addIndices(clusterState, strategy, 5, 1, 8);
+
+        logger.info(ShardAllocations.printShardDistribution(clusterState));
+        verifyPrimaryBalance(clusterState, buffer);
+
+        clusterState = removeOneNode(clusterState, strategy);
+
+        clusterState = applyAllocationUntilNoChange(clusterState, strategy);
+
+        logger.info(ShardAllocations.printShardDistribution(clusterState));
+        verifyPrimaryBalance(clusterState, buffer);
     }
 
     /**
@@ -537,7 +619,7 @@ private void verifyPerIndexPrimaryBalance(ClusterState currentState) {
         }
     }
 
-    private void verifyPrimaryBalance(ClusterState clusterState) throws Exception {
+    private void verifySkewedPrimaryBalance(ClusterState clusterState, int delta) throws Exception {
         assertBusy(() -> {
             RoutingNodes nodes = clusterState.getRoutingNodes();
             int totalPrimaryShards = 0;
@@ -545,13 +627,36 @@ private void verifyPrimaryBalance(ClusterState clusterState) throws Exception {
                 totalPrimaryShards += index.primaryShardsActive();
             }
             final int avgPrimaryShardsPerNode = (int) Math.ceil(totalPrimaryShards * 1f / clusterState.getRoutingNodes().size());
+            int maxPrimaryShardOnNode = Integer.MIN_VALUE;
+            int minPrimaryShardOnNode = Integer.MAX_VALUE;
             for (RoutingNode node : nodes) {
                 final int primaryCount = node.shardsWithState(STARTED)
                     .stream()
                     .filter(ShardRouting::primary)
                     .collect(Collectors.toList())
                     .size();
-                assertTrue(primaryCount <= avgPrimaryShardsPerNode);
+                maxPrimaryShardOnNode = Math.max(maxPrimaryShardOnNode, primaryCount);
+                minPrimaryShardOnNode = Math.min(minPrimaryShardOnNode, primaryCount);
+            }
+            assertTrue(maxPrimaryShardOnNode - minPrimaryShardOnNode < delta);
+        }, 60, TimeUnit.SECONDS);
+    }
+
+    private void verifyPrimaryBalance(ClusterState clusterState, float buffer) throws Exception {
+        assertBusy(() -> {
+            RoutingNodes nodes = clusterState.getRoutingNodes();
+            int totalPrimaryShards = 0;
+            for (final IndexRoutingTable index : clusterState.getRoutingTable().indicesRouting().values()) {
+                totalPrimaryShards += index.primaryShardsActive();
+            }
+            final int avgPrimaryShardsPerNode = (int) Math.ceil(totalPrimaryShards * 1f / clusterState.getRoutingNodes().size());
+            for (RoutingNode node : nodes) {
+                final int primaryCount = node.shardsWithState(STARTED)
+                    .stream()
+                    .filter(ShardRouting::primary)
+                    .collect(Collectors.toList())
+                    .size();
+                assertTrue(primaryCount <= (avgPrimaryShardsPerNode * (1 + buffer)));
             }
         }, 60, TimeUnit.SECONDS);
     }
@@ -567,8 +672,8 @@ public void testShardBalance() {
             ClusterRebalanceAllocationDecider.CLUSTER_ROUTING_ALLOCATION_ALLOW_REBALANCE_SETTING.getKey(),
             ClusterRebalanceAllocationDecider.ClusterRebalanceType.ALWAYS.toString()
         );
-        settings.put(BalancedShardsAllocator.INDEX_BALANCE_FACTOR_SETTING.getKey(), indexBalance);
         settings.put(BalancedShardsAllocator.SHARD_BALANCE_FACTOR_SETTING.getKey(), shardBalance);
+        settings.put(BalancedShardsAllocator.INDEX_BALANCE_FACTOR_SETTING.getKey(), indexBalance);
         settings.put(BalancedShardsAllocator.THRESHOLD_SETTING.getKey(), balanceThreshold);
 
         AllocationService strategy = createAllocationService(settings.build(), new TestGatewayAllocator());
@@ -634,6 +739,34 @@ private ClusterState addIndex(
         return applyAllocationUntilNoChange(clusterState, strategy);
     }
 
+    private ClusterState addIndices(
+        ClusterState clusterState,
+        AllocationService strategy,
+        int numberOfShards,
+        int numberOfReplicas,
+        int numberOfIndices
+    ) {
+        Metadata.Builder metadataBuilder = Metadata.builder(clusterState.getMetadata());
+        RoutingTable.Builder routingTableBuilder = RoutingTable.builder(clusterState.routingTable());
+
+        for (int i = 0; i < numberOfIndices; i++) {
+            IndexMetadata.Builder index = IndexMetadata.builder("test" + i)
+                .settings(settings(Version.CURRENT))
+                .numberOfShards(numberOfShards)
+                .numberOfReplicas(numberOfReplicas);
+
+            metadataBuilder = metadataBuilder.put(index);
+            routingTableBuilder.addAsNew(index.build());
+        }
+
+        clusterState = ClusterState.builder(clusterState)
+            .metadata(metadataBuilder.build())
+            .routingTable(routingTableBuilder.build())
+            .build();
+        clusterState = strategy.reroute(clusterState, "indices-created");
+        return applyAllocationUntilNoChange(clusterState, strategy);
+    }
+
     private ClusterState initCluster(
         AllocationService strategy,
         int numberOfIndices,
@@ -664,7 +797,7 @@ private ClusterState initCluster(
         for (int i = 0; i < numberOfNodes; i++) {
             nodes.add(newNode("node" + i));
         }
-        ClusterState clusterState = ClusterState.builder(org.opensearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
+        ClusterState clusterState = ClusterState.builder(CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .nodes(nodes)
             .metadata(metadata)
             .routingTable(initialRoutingTable)
@@ -673,6 +806,17 @@ private ClusterState initCluster(
         return applyAllocationUntilNoChange(clusterState, strategy);
     }
 
+    private ClusterState addNodes(ClusterState clusterState, AllocationService strategy, int numberOfNodes) {
+        logger.info("now, start [{}] more node, check that rebalancing will happen because we set it to always", numberOfNodes);
+        DiscoveryNodes.Builder nodes = DiscoveryNodes.builder(clusterState.nodes());
+        for (int i = 0; i < numberOfNodes; i++) {
+            nodes.add(newNode("node" + (clusterState.nodes().getSize() + i)));
+        }
+        clusterState = ClusterState.builder(clusterState).nodes(nodes.build()).build();
+        clusterState = strategy.reroute(clusterState, "reroute");
+        return applyStartedShardsUntilNoChange(clusterState, strategy);
+    }
+
     private ClusterState addNode(ClusterState clusterState, AllocationService strategy) {
         logger.info("now, start 1 more node, check that rebalancing will happen because we set it to always");
         clusterState = ClusterState.builder(clusterState)
@@ -918,7 +1062,7 @@ public ShardAllocationDecision decideShardAllocation(ShardRouting shard, Routing
             nodes.add(node);
         }
 
-        ClusterState clusterState = ClusterState.builder(org.opensearch.cluster.ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
+        ClusterState clusterState = ClusterState.builder(CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
             .nodes(nodes)
             .metadata(metadata)
             .routingTable(routingTable)
diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/FailedShardsRoutingTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/FailedShardsRoutingTests.java
index db4cedbbbe7b5..5e3b74ee138ab 100644
--- a/server/src/test/java/org/opensearch/cluster/routing/allocation/FailedShardsRoutingTests.java
+++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/FailedShardsRoutingTests.java
@@ -40,6 +40,7 @@
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.metadata.Metadata;
 import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.cluster.node.DiscoveryNodeRole;
 import org.opensearch.cluster.node.DiscoveryNodes;
 import org.opensearch.cluster.routing.RoutingNodes;
 import org.opensearch.cluster.routing.RoutingTable;
@@ -47,13 +48,18 @@
 import org.opensearch.cluster.routing.allocation.command.AllocationCommands;
 import org.opensearch.cluster.routing.allocation.command.MoveAllocationCommand;
 import org.opensearch.cluster.routing.allocation.decider.ClusterRebalanceAllocationDecider;
+import org.opensearch.common.UUIDs;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.core.index.shard.ShardId;
 import org.opensearch.indices.replication.common.ReplicationType;
+import org.opensearch.node.remotestore.RemoteStoreNodeService;
 import org.opensearch.test.VersionUtils;
 
 import java.util.ArrayList;
 import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
 import java.util.Set;
 
 import static org.opensearch.cluster.ClusterName.CLUSTER_NAME_SETTING;
@@ -61,6 +67,11 @@
 import static org.opensearch.cluster.routing.ShardRoutingState.RELOCATING;
 import static org.opensearch.cluster.routing.ShardRoutingState.STARTED;
 import static org.opensearch.cluster.routing.ShardRoutingState.UNASSIGNED;
+import static org.opensearch.common.util.FeatureFlags.REMOTE_STORE_MIGRATION_EXPERIMENTAL;
+import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY;
+import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY;
+import static org.opensearch.node.remotestore.RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING;
+import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING;
 import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.lessThan;
@@ -812,4 +823,136 @@ private void testReplicaIsPromoted(boolean isSegmentReplicationEnabled) {
             }
         }
     }
+
+    public void testPreferReplicaOnRemoteNodeForPrimaryPromotion() {
+        FeatureFlags.initializeFeatureFlags(Settings.builder().put(REMOTE_STORE_MIGRATION_EXPERIMENTAL, "true").build());
+        AllocationService allocation = createAllocationService(Settings.builder().build());
+
+        // segment replication enabled
+        Settings.Builder settingsBuilder = settings(Version.CURRENT).put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT);
+
+        // remote store migration metadata settings
+        Metadata metadata = Metadata.builder()
+            .put(IndexMetadata.builder("test").settings(settingsBuilder).numberOfShards(1).numberOfReplicas(4))
+            .persistentSettings(
+                Settings.builder()
+                    .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), RemoteStoreNodeService.CompatibilityMode.MIXED.mode)
+                    .put(MIGRATION_DIRECTION_SETTING.getKey(), RemoteStoreNodeService.Direction.REMOTE_STORE.direction)
+                    .build()
+            )
+            .build();
+
+        RoutingTable initialRoutingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+
+        ClusterState clusterState = ClusterState.builder(CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
+            .metadata(metadata)
+            .routingTable(initialRoutingTable)
+            .build();
+
+        ShardId shardId = new ShardId(metadata.index("test").getIndex(), 0);
+
+        // add a remote node and start primary shard
+        Map<String, String> remoteStoreNodeAttributes = Map.of(
+            REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY,
+            "REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_VALUE",
+            REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY,
+            "REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_VALUE"
+        );
+        DiscoveryNode remoteNode1 = new DiscoveryNode(
+            UUIDs.base64UUID(),
+            buildNewFakeTransportAddress(),
+            remoteStoreNodeAttributes,
+            DiscoveryNodeRole.BUILT_IN_ROLES,
+            Version.CURRENT
+        );
+        clusterState = ClusterState.builder(clusterState).nodes(DiscoveryNodes.builder().add(remoteNode1)).build();
+        clusterState = ClusterState.builder(clusterState).routingTable(allocation.reroute(clusterState, "reroute").routingTable()).build();
+        assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(1));
+        assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).size(), equalTo(4));
+
+        clusterState = startInitializingShardsAndReroute(allocation, clusterState);
+        assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(1));
+        assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).size(), equalTo(4));
+
+        // add remote and non-remote nodes and start replica shards
+        DiscoveryNode remoteNode2 = new DiscoveryNode(
+            UUIDs.base64UUID(),
+            buildNewFakeTransportAddress(),
+            remoteStoreNodeAttributes,
+            DiscoveryNodeRole.BUILT_IN_ROLES,
+            Version.CURRENT
+        );
+        DiscoveryNode remoteNode3 = new DiscoveryNode(
+            UUIDs.base64UUID(),
+            buildNewFakeTransportAddress(),
+            remoteStoreNodeAttributes,
+            DiscoveryNodeRole.BUILT_IN_ROLES,
+            Version.CURRENT
+        );
+        DiscoveryNode nonRemoteNode1 = new DiscoveryNode(UUIDs.base64UUID(), buildNewFakeTransportAddress(), Version.CURRENT);
+        DiscoveryNode nonRemoteNode2 = new DiscoveryNode(UUIDs.base64UUID(), buildNewFakeTransportAddress(), Version.CURRENT);
+        List<DiscoveryNode> replicaShardNodes = List.of(remoteNode2, remoteNode3, nonRemoteNode1, nonRemoteNode2);
+
+        for (int i = 0; i < 4; i++) {
+            clusterState = ClusterState.builder(clusterState)
+                .nodes(DiscoveryNodes.builder(clusterState.nodes()).add(replicaShardNodes.get(i)))
+                .build();
+
+            clusterState = allocation.reroute(clusterState, "reroute");
+            assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(1 + i));
+            assertThat(clusterState.getRoutingNodes().shardsWithState(INITIALIZING).size(), equalTo(1));
+            assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).size(), equalTo(4 - (i + 1)));
+
+            clusterState = startInitializingShardsAndReroute(allocation, clusterState);
+            assertThat(clusterState.getRoutingNodes().shardsWithState(STARTED).size(), equalTo(1 + (i + 1)));
+            assertThat(clusterState.getRoutingNodes().shardsWithState(UNASSIGNED).size(), equalTo(4 - (i + 1)));
+        }
+
+        // fail primary shard
+        ShardRouting primaryShard0 = clusterState.routingTable().index("test").shard(0).primaryShard();
+        ClusterState newState = allocation.applyFailedShard(clusterState, primaryShard0, randomBoolean());
+        assertNotEquals(clusterState, newState);
+        clusterState = newState;
+
+        // verify that promoted replica exists on a remote node
+        assertEquals(4, clusterState.getRoutingNodes().shardsWithState(STARTED).size());
+        ShardRouting primaryShardRouting1 = clusterState.routingTable().index("test").shard(0).primaryShard();
+        assertNotEquals(primaryShard0, primaryShardRouting1);
+        assertTrue(
+            primaryShardRouting1.currentNodeId().equals(remoteNode2.getId())
+                || primaryShardRouting1.currentNodeId().equals(remoteNode3.getId())
+        );
+
+        // fail primary shard again
+        newState = allocation.applyFailedShard(clusterState, primaryShardRouting1, randomBoolean());
+        assertNotEquals(clusterState, newState);
+        clusterState = newState;
+
+        // verify that promoted replica again exists on a remote node
+        assertEquals(3, clusterState.getRoutingNodes().shardsWithState(STARTED).size());
+        ShardRouting primaryShardRouting2 = clusterState.routingTable().index("test").shard(0).primaryShard();
+        assertNotEquals(primaryShardRouting1, primaryShardRouting2);
+        assertTrue(
+            primaryShardRouting2.currentNodeId().equals(remoteNode2.getId())
+                || primaryShardRouting2.currentNodeId().equals(remoteNode3.getId())
+        );
+        assertNotEquals(primaryShardRouting1.currentNodeId(), primaryShardRouting2.currentNodeId());
+
+        ShardRouting expectedCandidateForSegRep = clusterState.getRoutingNodes().activeReplicaWithOldestVersion(shardId);
+
+        // fail primary shard again
+        newState = allocation.applyFailedShard(clusterState, primaryShardRouting2, randomBoolean());
+        assertNotEquals(clusterState, newState);
+        clusterState = newState;
+
+        // verify that promoted replica exists on a non-remote node
+        assertEquals(2, clusterState.getRoutingNodes().shardsWithState(STARTED).size());
+        ShardRouting primaryShardRouting3 = clusterState.routingTable().index("test").shard(0).primaryShard();
+        assertNotEquals(primaryShardRouting2, primaryShardRouting3);
+        assertTrue(
+            primaryShardRouting3.currentNodeId().equals(nonRemoteNode1.getId())
+                || primaryShardRouting3.currentNodeId().equals(nonRemoteNode2.getId())
+        );
+        assertEquals(expectedCandidateForSegRep.allocationId(), primaryShardRouting3.allocationId());
+    }
 }
diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/RemoteStoreMigrationAllocationDeciderTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/RemoteStoreMigrationAllocationDeciderTests.java
index 43363407d9249..ee4dbe9738e04 100644
--- a/server/src/test/java/org/opensearch/cluster/routing/allocation/RemoteStoreMigrationAllocationDeciderTests.java
+++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/RemoteStoreMigrationAllocationDeciderTests.java
@@ -70,6 +70,8 @@
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE;
 import static org.opensearch.common.util.FeatureFlags.REMOTE_STORE_MIGRATION_EXPERIMENTAL;
 import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY;
+import static org.opensearch.node.remotestore.RemoteStoreNodeService.Direction.NONE;
+import static org.opensearch.node.remotestore.RemoteStoreNodeService.Direction.REMOTE_STORE;
 import static org.opensearch.node.remotestore.RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING;
 import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING;
 import static org.hamcrest.core.Is.is;
@@ -89,7 +91,7 @@ public class RemoteStoreMigrationAllocationDeciderTests extends OpenSearchAlloca
         .build();
 
     private final Settings remoteStoreDirectionSettings = Settings.builder()
-        .put(MIGRATION_DIRECTION_SETTING.getKey(), RemoteStoreNodeService.Direction.REMOTE_STORE)
+        .put(MIGRATION_DIRECTION_SETTING.getKey(), REMOTE_STORE)
         .build();
     private final Settings docrepDirectionSettings = Settings.builder()
         .put(MIGRATION_DIRECTION_SETTING.getKey(), RemoteStoreNodeService.Direction.DOCREP)
@@ -106,7 +108,9 @@ public class RemoteStoreMigrationAllocationDeciderTests extends OpenSearchAlloca
     private Metadata metadata;
     private RoutingTable routingTable = null;
 
-    private void beforeAllocation() {
+    private ShardId shardId = new ShardId(TEST_INDEX, "_na_", 0);
+
+    private void beforeAllocation(String direction) {
         FeatureFlags.initializeFeatureFlags(directionEnabledNodeSettings);
         if (isRemoteStoreBackedIndex == null) {
             isRemoteStoreBackedIndex = randomBoolean();
@@ -116,11 +120,7 @@ private void beforeAllocation() {
         String compatibilityMode = isMixedMode
             ? RemoteStoreNodeService.CompatibilityMode.MIXED.mode
             : RemoteStoreNodeService.CompatibilityMode.STRICT.mode;
-        customSettings = getCustomSettings(
-            RemoteStoreNodeService.Direction.REMOTE_STORE.direction,
-            compatibilityMode,
-            indexMetadataBuilder
-        );
+        customSettings = getCustomSettings(direction, compatibilityMode, indexMetadataBuilder);
 
         if (routingTable != null) {
             metadata = Metadata.builder().put(indexMetadataBuilder).build();
@@ -149,6 +149,35 @@ private void beforeAllocation() {
         routingAllocation.debugDecision(true);
     }
 
+    private void prepareRoutingTable(boolean isReplicaAllocation, String primaryShardNodeId) {
+        routingTable = RoutingTable.builder()
+            .add(
+                IndexRoutingTable.builder(shardId.getIndex())
+                    .addIndexShard(
+                        new IndexShardRoutingTable.Builder(shardId).addShard(
+                            TestShardRouting.newShardRouting(
+                                shardId.getIndexName(),
+                                shardId.getId(),
+                                (isReplicaAllocation ? primaryShardNodeId : null),
+                                true,
+                                (isReplicaAllocation ? ShardRoutingState.STARTED : ShardRoutingState.UNASSIGNED)
+                            )
+                        )
+                            .addShard(
+                                TestShardRouting.newShardRouting(
+                                    shardId.getIndexName(),
+                                    shardId.getId(),
+                                    null,
+                                    false,
+                                    ShardRoutingState.UNASSIGNED
+                                )
+                            )
+                            .build()
+                    )
+            )
+            .build();
+    }
+
     // tests for primary shard copy allocation with MIXED mode and REMOTE_STORE direction
 
     public void testDontAllocateNewPrimaryShardOnNonRemoteNodeForMixedModeAndRemoteStoreDirection() {
@@ -166,7 +195,7 @@ public void testDontAllocateNewPrimaryShardOnNonRemoteNodeForMixedModeAndRemoteS
             .localNodeId(remoteNode.getId())
             .build();
 
-        beforeAllocation();
+        beforeAllocation(REMOTE_STORE.direction);
 
         ShardRouting primaryShardRouting = clusterState.getRoutingTable().shardRoutingTable(TEST_INDEX, 0).primaryShard();
         RoutingNode nonRemoteRoutingNode = clusterState.getRoutingNodes().node(nonRemoteNode.getId());
@@ -196,7 +225,7 @@ public void testAllocateNewPrimaryShardOnRemoteNodeForMixedModeAndRemoteStoreDir
             .localNodeId(remoteNode.getId())
             .build();
 
-        beforeAllocation();
+        beforeAllocation(REMOTE_STORE.direction);
 
         ShardRouting primaryShardRouting = clusterState.getRoutingTable().shardRoutingTable(TEST_INDEX, 0).primaryShard();
         RoutingNode remoteRoutingNode = clusterState.getRoutingNodes().node(remoteNode.getId());
@@ -216,39 +245,11 @@ public void testDontAllocateNewReplicaShardOnRemoteNodeIfPrimaryShardOnNonRemote
         replicaCount = 1;
         isMixedMode = true;
 
-        ShardId shardId = new ShardId(TEST_INDEX, "_na_", 0);
-
         DiscoveryNode nonRemoteNode = getNonRemoteNode();
         DiscoveryNode remoteNode = getRemoteNode();
 
-        routingTable = RoutingTable.builder()
-            .add(
-                IndexRoutingTable.builder(shardId.getIndex())
-                    .addIndexShard(
-                        new IndexShardRoutingTable.Builder(shardId).addShard(
-                            // primary on non-remote node
-                            TestShardRouting.newShardRouting(
-                                shardId.getIndexName(),
-                                shardId.getId(),
-                                nonRemoteNode.getId(),
-                                true,
-                                ShardRoutingState.STARTED
-                            )
-                        )
-                            .addShard(
-                                // new replica's allocation
-                                TestShardRouting.newShardRouting(
-                                    shardId.getIndexName(),
-                                    shardId.getId(),
-                                    null,
-                                    false,
-                                    ShardRoutingState.UNASSIGNED
-                                )
-                            )
-                            .build()
-                    )
-            )
-            .build();
+        // primary on non-remote node, new replica's allocation
+        prepareRoutingTable(true, nonRemoteNode.getId());
 
         discoveryNodes = DiscoveryNodes.builder()
             .add(nonRemoteNode)
@@ -257,7 +258,7 @@ public void testDontAllocateNewReplicaShardOnRemoteNodeIfPrimaryShardOnNonRemote
             .localNodeId(remoteNode.getId())
             .build();
 
-        beforeAllocation();
+        beforeAllocation(REMOTE_STORE.direction);
 
         assertEquals(2, clusterState.getRoutingTable().allShards().size());
         ShardRouting replicaShardRouting = clusterState.getRoutingTable().shardRoutingTable(TEST_INDEX, 0).replicaShards().get(0);
@@ -278,40 +279,12 @@ public void testAllocateNewReplicaShardOnRemoteNodeIfPrimaryShardOnRemoteNodeFor
         replicaCount = 1;
         isMixedMode = true;
 
-        ShardId shardId = new ShardId(TEST_INDEX, "_na_", 0);
-
         DiscoveryNode remoteNode1 = getRemoteNode();
         DiscoveryNode remoteNode2 = getRemoteNode();
         DiscoveryNode nonRemoteNode = getNonRemoteNode();
 
-        routingTable = RoutingTable.builder()
-            .add(
-                IndexRoutingTable.builder(shardId.getIndex())
-                    .addIndexShard(
-                        new IndexShardRoutingTable.Builder(shardId).addShard(
-                            // primary on remote node
-                            TestShardRouting.newShardRouting(
-                                shardId.getIndexName(),
-                                shardId.getId(),
-                                remoteNode1.getId(),
-                                true,
-                                ShardRoutingState.STARTED
-                            )
-                        )
-                            .addShard(
-                                // new replica's allocation
-                                TestShardRouting.newShardRouting(
-                                    shardId.getIndexName(),
-                                    shardId.getId(),
-                                    null,
-                                    false,
-                                    ShardRoutingState.UNASSIGNED
-                                )
-                            )
-                            .build()
-                    )
-            )
-            .build();
+        // primary on remote node, new replica's allocation
+        prepareRoutingTable(true, remoteNode1.getId());
 
         discoveryNodes = DiscoveryNodes.builder()
             .add(remoteNode1)
@@ -322,7 +295,7 @@ public void testAllocateNewReplicaShardOnRemoteNodeIfPrimaryShardOnRemoteNodeFor
             .localNodeId(nonRemoteNode.getId())
             .build();
 
-        beforeAllocation();
+        beforeAllocation(REMOTE_STORE.direction);
 
         assertEquals(2, clusterState.getRoutingTable().allShards().size());
         ShardRouting replicaShardRouting = clusterState.getRoutingTable().shardRoutingTable(TEST_INDEX, 0).replicaShards().get(0);
@@ -343,40 +316,12 @@ public void testAllocateNewReplicaShardOnNonRemoteNodeIfPrimaryShardOnNonRemoteN
         replicaCount = 1;
         isMixedMode = true;
 
-        ShardId shardId = new ShardId(TEST_INDEX, "_na_", 0);
-
         DiscoveryNode remoteNode = getRemoteNode();
         DiscoveryNode nonRemoteNode1 = getNonRemoteNode();
         DiscoveryNode nonRemoteNode2 = getNonRemoteNode();
 
-        routingTable = RoutingTable.builder()
-            .add(
-                IndexRoutingTable.builder(shardId.getIndex())
-                    .addIndexShard(
-                        new IndexShardRoutingTable.Builder(shardId).addShard(
-                            // primary shard on non-remote node
-                            TestShardRouting.newShardRouting(
-                                shardId.getIndexName(),
-                                shardId.getId(),
-                                nonRemoteNode1.getId(),
-                                true,
-                                ShardRoutingState.STARTED
-                            )
-                        )
-                            .addShard(
-                                // new replica's allocation
-                                TestShardRouting.newShardRouting(
-                                    shardId.getIndexName(),
-                                    shardId.getId(),
-                                    null,
-                                    false,
-                                    ShardRoutingState.UNASSIGNED
-                                )
-                            )
-                            .build()
-                    )
-            )
-            .build();
+        // primary shard on non-remote node, new replica's allocation
+        prepareRoutingTable(true, nonRemoteNode1.getId());
 
         discoveryNodes = DiscoveryNodes.builder()
             .add(remoteNode)
@@ -387,7 +332,7 @@ public void testAllocateNewReplicaShardOnNonRemoteNodeIfPrimaryShardOnNonRemoteN
             .localNodeId(nonRemoteNode2.getId())
             .build();
 
-        beforeAllocation();
+        beforeAllocation(REMOTE_STORE.direction);
 
         assertEquals(2, clusterState.getRoutingTable().allShards().size());
 
@@ -411,39 +356,11 @@ public void testAllocateNewReplicaShardOnNonRemoteNodeIfPrimaryShardOnRemoteNode
         replicaCount = 1;
         isMixedMode = true;
 
-        ShardId shardId = new ShardId(TEST_INDEX, "_na_", 0);
-
         DiscoveryNode nonRemoteNode = getNonRemoteNode();
         DiscoveryNode remoteNode = getRemoteNode();
 
-        routingTable = RoutingTable.builder()
-            .add(
-                IndexRoutingTable.builder(shardId.getIndex())
-                    .addIndexShard(
-                        new IndexShardRoutingTable.Builder(shardId).addShard(
-                            // primary shard on non-remote node
-                            TestShardRouting.newShardRouting(
-                                shardId.getIndexName(),
-                                shardId.getId(),
-                                remoteNode.getId(),
-                                true,
-                                ShardRoutingState.STARTED
-                            )
-                        )
-                            .addShard(
-                                // new replica's allocation
-                                TestShardRouting.newShardRouting(
-                                    shardId.getIndexName(),
-                                    shardId.getId(),
-                                    null,
-                                    false,
-                                    ShardRoutingState.UNASSIGNED
-                                )
-                            )
-                            .build()
-                    )
-            )
-            .build();
+        // primary shard on remote node, new replica's allocation
+        prepareRoutingTable(true, remoteNode.getId());
 
         discoveryNodes = DiscoveryNodes.builder()
             .add(nonRemoteNode)
@@ -452,7 +369,7 @@ public void testAllocateNewReplicaShardOnNonRemoteNodeIfPrimaryShardOnRemoteNode
             .localNodeId(remoteNode.getId())
             .build();
 
-        beforeAllocation();
+        beforeAllocation(REMOTE_STORE.direction);
 
         assertEquals(2, clusterState.getRoutingTable().allShards().size());
         ShardRouting replicaShardRouting = clusterState.getRoutingTable().shardRoutingTable(TEST_INDEX, 0).replicaShards().get(0);
@@ -478,39 +395,12 @@ public void testAlwaysAllocateNewShardForStrictMode() {
         isMixedMode = false;
         isRemoteStoreBackedIndex = false;
 
-        ShardId shardId = new ShardId(TEST_INDEX, "_na_", 0);
-
         DiscoveryNode nonRemoteNode1 = getNonRemoteNode();
         DiscoveryNode nonRemoteNode2 = getNonRemoteNode();
 
         boolean isReplicaAllocation = randomBoolean();
 
-        routingTable = RoutingTable.builder()
-            .add(
-                IndexRoutingTable.builder(shardId.getIndex())
-                    .addIndexShard(
-                        new IndexShardRoutingTable.Builder(shardId).addShard(
-                            TestShardRouting.newShardRouting(
-                                shardId.getIndexName(),
-                                shardId.getId(),
-                                (isReplicaAllocation ? nonRemoteNode1.getId() : null),
-                                true,
-                                (isReplicaAllocation ? ShardRoutingState.STARTED : ShardRoutingState.UNASSIGNED)
-                            )
-                        )
-                            .addShard(
-                                TestShardRouting.newShardRouting(
-                                    shardId.getIndexName(),
-                                    shardId.getId(),
-                                    null,
-                                    false,
-                                    ShardRoutingState.UNASSIGNED
-                                )
-                            )
-                            .build()
-                    )
-            )
-            .build();
+        prepareRoutingTable(isReplicaAllocation, nonRemoteNode1.getId());
 
         discoveryNodes = DiscoveryNodes.builder()
             .add(nonRemoteNode1)
@@ -519,7 +409,7 @@ public void testAlwaysAllocateNewShardForStrictMode() {
             .localNodeId(nonRemoteNode2.getId())
             .build();
 
-        beforeAllocation();
+        beforeAllocation(REMOTE_STORE.direction);
 
         assertEquals(2, clusterState.getRoutingTable().allShards().size());
 
@@ -543,33 +433,7 @@ public void testAlwaysAllocateNewShardForStrictMode() {
         DiscoveryNode remoteNode1 = getRemoteNode();
         DiscoveryNode remoteNode2 = getRemoteNode();
 
-        routingTable = RoutingTable.builder()
-            .add(
-                IndexRoutingTable.builder(shardId.getIndex())
-                    .addIndexShard(
-                        new IndexShardRoutingTable.Builder(shardId).addShard(
-                            TestShardRouting.newShardRouting(
-                                shardId.getIndexName(),
-                                shardId.getId(),
-                                (isReplicaAllocation ? remoteNode1.getId() : null),
-                                true,
-                                (isReplicaAllocation ? ShardRoutingState.STARTED : ShardRoutingState.UNASSIGNED)
-                            )
-                        )
-                            .addShard(
-                                // new replica's allocation
-                                TestShardRouting.newShardRouting(
-                                    shardId.getIndexName(),
-                                    shardId.getId(),
-                                    null,
-                                    false,
-                                    ShardRoutingState.UNASSIGNED
-                                )
-                            )
-                            .build()
-                    )
-            )
-            .build();
+        prepareRoutingTable(isReplicaAllocation, remoteNode1.getId());
 
         discoveryNodes = DiscoveryNodes.builder()
             .add(remoteNode1)
@@ -578,7 +442,7 @@ public void testAlwaysAllocateNewShardForStrictMode() {
             .localNodeId(remoteNode2.getId())
             .build();
 
-        beforeAllocation();
+        beforeAllocation(REMOTE_STORE.direction);
 
         assertEquals(2, clusterState.getRoutingTable().allShards().size());
 
@@ -598,6 +462,97 @@ public void testAlwaysAllocateNewShardForStrictMode() {
         assertThat(decision.getExplanation().toLowerCase(Locale.ROOT), is(reason));
     }
 
+    // test for NONE direction
+    public void testAllocationForNoneDirection() {
+        shardCount = 1;
+        replicaCount = 1;
+        isMixedMode = true;
+        isRemoteStoreBackedIndex = false; // non-remote store backed index
+
+        DiscoveryNode remoteNode1 = getRemoteNode();
+        DiscoveryNode remoteNode2 = getRemoteNode();
+        DiscoveryNode nonRemoteNode1 = getNonRemoteNode();
+        DiscoveryNode nonRemoteNode2 = getNonRemoteNode();
+
+        boolean isReplicaAllocation = randomBoolean();
+
+        prepareRoutingTable(isReplicaAllocation, nonRemoteNode1.getId());
+
+        discoveryNodes = DiscoveryNodes.builder()
+            .add(remoteNode1)
+            .localNodeId(remoteNode1.getId())
+            .add(remoteNode2)
+            .localNodeId(remoteNode2.getId())
+            .add(nonRemoteNode1)
+            .localNodeId(nonRemoteNode1.getId())
+            .add(nonRemoteNode2)
+            .localNodeId(nonRemoteNode2.getId())
+            .build();
+
+        beforeAllocation(NONE.direction);
+        assertEquals(2, clusterState.getRoutingTable().allShards().size());
+
+        ShardRouting shardRouting = clusterState.getRoutingTable().shardRoutingTable(TEST_INDEX, 0).primaryShard();
+        if (isReplicaAllocation) {
+            shardRouting = clusterState.getRoutingTable().shardRoutingTable(TEST_INDEX, 0).replicaShards().get(0);
+        }
+        RoutingNode nonRemoteRoutingNode = clusterState.getRoutingNodes().node(nonRemoteNode2.getId());
+        RoutingNode remoteRoutingNode = clusterState.getRoutingNodes().node(remoteNode2.getId());
+
+        // allocation decision for non-remote node for non-remote store backed index
+        Decision decision = remoteStoreMigrationAllocationDecider.canAllocate(shardRouting, nonRemoteRoutingNode, routingAllocation);
+        assertThat(decision.type(), is(Decision.Type.YES));
+        String reason = String.format(
+            Locale.ROOT,
+            "[none migration_direction]: %s shard copy can be allocated to a non-remote node for non remote store backed index",
+            (isReplicaAllocation ? "replica" : "primary")
+        );
+        assertThat(decision.getExplanation().toLowerCase(Locale.ROOT), is(reason));
+
+        // allocation decision for remote node for non-remote store backed index
+        decision = remoteStoreMigrationAllocationDecider.canAllocate(shardRouting, remoteRoutingNode, routingAllocation);
+        assertThat(decision.type(), is(Decision.Type.NO));
+        reason = String.format(
+            Locale.ROOT,
+            "[none migration_direction]: %s shard copy can not be allocated to a remote node for non remote store backed index",
+            (isReplicaAllocation ? "replica" : "primary")
+        );
+        assertThat(decision.getExplanation().toLowerCase(Locale.ROOT), is(reason));
+
+        isRemoteStoreBackedIndex = true; // remote store backed index
+        prepareRoutingTable(isReplicaAllocation, remoteNode1.getId());
+
+        beforeAllocation(NONE.direction);
+        assertEquals(2, clusterState.getRoutingTable().allShards().size());
+
+        shardRouting = clusterState.getRoutingTable().shardRoutingTable(TEST_INDEX, 0).primaryShard();
+        if (isReplicaAllocation) {
+            shardRouting = clusterState.getRoutingTable().shardRoutingTable(TEST_INDEX, 0).replicaShards().get(0);
+        }
+        nonRemoteRoutingNode = clusterState.getRoutingNodes().node(nonRemoteNode2.getId());
+        remoteRoutingNode = clusterState.getRoutingNodes().node(remoteNode2.getId());
+
+        // allocation decision for non-remote node for remote store backed index
+        decision = remoteStoreMigrationAllocationDecider.canAllocate(shardRouting, nonRemoteRoutingNode, routingAllocation);
+        assertThat(decision.type(), is(Decision.Type.NO));
+        reason = String.format(
+            Locale.ROOT,
+            "[none migration_direction]: %s shard copy can not be allocated to a non-remote node for remote store backed index",
+            (isReplicaAllocation ? "replica" : "primary")
+        );
+        assertThat(decision.getExplanation().toLowerCase(Locale.ROOT), is(reason));
+
+        // allocation decision for remote node for remote store backed index
+        decision = remoteStoreMigrationAllocationDecider.canAllocate(shardRouting, remoteRoutingNode, routingAllocation);
+        assertThat(decision.type(), is(Decision.Type.YES));
+        reason = String.format(
+            Locale.ROOT,
+            "[none migration_direction]: %s shard copy can be allocated to a remote node for remote store backed index",
+            (isReplicaAllocation ? "replica" : "primary")
+        );
+        assertThat(decision.getExplanation().toLowerCase(Locale.ROOT), is(reason));
+    }
+
     // prepare index metadata for test-index
     private IndexMetadata.Builder getIndexMetadataBuilder(boolean isRemoteStoreBackedIndex, int shardCount, int replicaCount) {
         Settings.Builder builder = settings(Version.CURRENT);
@@ -614,7 +569,7 @@ private IndexMetadata.Builder getIndexMetadataBuilder(boolean isRemoteStoreBacke
     private Settings getCustomSettings(String direction, String compatibilityMode, IndexMetadata.Builder indexMetadataBuilder) {
         Settings.Builder builder = Settings.builder();
         // direction settings
-        if (direction.toLowerCase(Locale.ROOT).equals(RemoteStoreNodeService.Direction.REMOTE_STORE.direction)) {
+        if (direction.toLowerCase(Locale.ROOT).equals(REMOTE_STORE.direction)) {
             builder.put(remoteStoreDirectionSettings);
         } else if (direction.toLowerCase(Locale.ROOT).equals(RemoteStoreNodeService.Direction.DOCREP.direction)) {
             builder.put(docrepDirectionSettings);
@@ -664,7 +619,7 @@ private DiscoveryNode getNonRemoteNode() {
     }
 
     // get a dummy remote node
-    public DiscoveryNode getRemoteNode() {
+    private DiscoveryNode getRemoteNode() {
         Map<String, String> attributes = new HashMap<>();
         attributes.put(
             REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY,
diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/FilterAllocationDeciderTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/FilterAllocationDeciderTests.java
index a8282faaddced..e8273d294f24f 100644
--- a/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/FilterAllocationDeciderTests.java
+++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/decider/FilterAllocationDeciderTests.java
@@ -42,6 +42,7 @@
 import org.opensearch.cluster.routing.RecoverySource;
 import org.opensearch.cluster.routing.RoutingTable;
 import org.opensearch.cluster.routing.ShardRouting;
+import org.opensearch.cluster.routing.UnassignedInfo;
 import org.opensearch.cluster.routing.allocation.AllocationService;
 import org.opensearch.cluster.routing.allocation.RoutingAllocation;
 import org.opensearch.cluster.routing.allocation.allocator.BalancedShardsAllocator;
@@ -50,6 +51,8 @@
 import org.opensearch.common.settings.IndexScopedSettings;
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.FeatureFlags;
+import org.opensearch.node.remotestore.RemoteStoreNodeService;
 import org.opensearch.snapshots.EmptySnapshotsInfoService;
 import org.opensearch.test.gateway.TestGatewayAllocator;
 
@@ -61,6 +64,9 @@
 import static org.opensearch.cluster.routing.ShardRoutingState.INITIALIZING;
 import static org.opensearch.cluster.routing.ShardRoutingState.STARTED;
 import static org.opensearch.cluster.routing.ShardRoutingState.UNASSIGNED;
+import static org.opensearch.common.util.FeatureFlags.REMOTE_STORE_MIGRATION_EXPERIMENTAL;
+import static org.opensearch.node.remotestore.RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING;
+import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING;
 
 public class FilterAllocationDeciderTests extends OpenSearchAllocationTestCase {
 
@@ -406,4 +412,57 @@ public void testWildcardIPFilter() {
             "test ip validation"
         );
     }
+
+    public void testMixedModeRemoteStoreAllocation() {
+        // For mixed mode remote store direction cluster's existing indices replica creation ,
+        // we don't consider filter allocation decider for replica of existing indices
+        FeatureFlags.initializeFeatureFlags(Settings.builder().put(REMOTE_STORE_MIGRATION_EXPERIMENTAL, "true").build());
+        ClusterSettings clusterSettings = new ClusterSettings(Settings.builder().build(), ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        Settings initialSettings = Settings.builder()
+            .put("cluster.routing.allocation.exclude._id", "node2")
+            .put(REMOTE_STORE_COMPATIBILITY_MODE_SETTING.getKey(), RemoteStoreNodeService.CompatibilityMode.MIXED)
+            .put(MIGRATION_DIRECTION_SETTING.getKey(), RemoteStoreNodeService.Direction.REMOTE_STORE)
+            .build();
+
+        FilterAllocationDecider filterAllocationDecider = new FilterAllocationDecider(initialSettings, clusterSettings);
+        AllocationDeciders allocationDeciders = new AllocationDeciders(
+            Arrays.asList(
+                filterAllocationDecider,
+                new SameShardAllocationDecider(Settings.EMPTY, clusterSettings),
+                new ReplicaAfterPrimaryActiveAllocationDecider()
+            )
+        );
+        AllocationService service = new AllocationService(
+            allocationDeciders,
+            new TestGatewayAllocator(),
+            new BalancedShardsAllocator(Settings.EMPTY),
+            EmptyClusterInfoService.INSTANCE,
+            EmptySnapshotsInfoService.INSTANCE
+        );
+        ClusterState state = createInitialClusterState(service, Settings.EMPTY, Settings.EMPTY);
+        RoutingTable routingTable = state.routingTable();
+        RoutingAllocation allocation = new RoutingAllocation(allocationDeciders, state.getRoutingNodes(), state, null, null, 0);
+        allocation.debugDecision(true);
+        ShardRouting sr = ShardRouting.newUnassigned(
+            routingTable.index("sourceIndex").shard(0).shardId(),
+            false,
+            RecoverySource.PeerRecoverySource.INSTANCE,
+            new UnassignedInfo(UnassignedInfo.Reason.NODE_LEFT, "")
+        );
+        Decision.Single decision = (Decision.Single) filterAllocationDecider.canAllocate(
+            sr,
+            state.getRoutingNodes().node("node2"),
+            allocation
+        );
+        assertEquals(decision.toString(), Type.YES, decision.type());
+
+        sr = ShardRouting.newUnassigned(
+            routingTable.index("sourceIndex").shard(0).shardId(),
+            false,
+            RecoverySource.PeerRecoverySource.INSTANCE,
+            new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "")
+        );
+        decision = (Decision.Single) filterAllocationDecider.canAllocate(sr, state.getRoutingNodes().node("node2"), allocation);
+        assertEquals(decision.toString(), Type.NO, decision.type());
+    }
 }
diff --git a/server/src/test/java/org/opensearch/cluster/service/ClusterApplierServiceTests.java b/server/src/test/java/org/opensearch/cluster/service/ClusterApplierServiceTests.java
index 1c70d18800caa..3cbdfb80067d7 100644
--- a/server/src/test/java/org/opensearch/cluster/service/ClusterApplierServiceTests.java
+++ b/server/src/test/java/org/opensearch/cluster/service/ClusterApplierServiceTests.java
@@ -35,6 +35,7 @@
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.opensearch.Version;
+import org.opensearch.cluster.ClusterManagerMetrics;
 import org.opensearch.cluster.ClusterName;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.ClusterStateObserver;
@@ -135,7 +136,7 @@ private TimedClusterApplierService createTimedClusterService(boolean makeCluster
             Settings.builder().put("cluster.name", "ClusterApplierServiceTests").build(),
             new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
             threadPool,
-            metricsRegistry
+            new ClusterManagerMetrics(metricsRegistry)
         );
         timedClusterApplierService.setNodeConnectionsService(createNoOpNodeConnectionsService());
         timedClusterApplierService.setInitialState(
@@ -678,9 +679,9 @@ static class TimedClusterApplierService extends ClusterApplierService {
             Settings settings,
             ClusterSettings clusterSettings,
             ThreadPool threadPool,
-            MetricsRegistry metricsRegistry
+            ClusterManagerMetrics clusterManagerMetrics
         ) {
-            super("test_node", settings, clusterSettings, threadPool, metricsRegistry);
+            super("test_node", settings, clusterSettings, threadPool, clusterManagerMetrics);
             this.clusterSettings = clusterSettings;
         }
 
diff --git a/server/src/test/java/org/opensearch/cluster/service/MasterServiceTests.java b/server/src/test/java/org/opensearch/cluster/service/MasterServiceTests.java
index e58bde8e0bc77..0ff8d9dc4e7a5 100644
--- a/server/src/test/java/org/opensearch/cluster/service/MasterServiceTests.java
+++ b/server/src/test/java/org/opensearch/cluster/service/MasterServiceTests.java
@@ -40,6 +40,7 @@
 import org.opensearch.Version;
 import org.opensearch.cluster.AckedClusterStateUpdateTask;
 import org.opensearch.cluster.ClusterChangedEvent;
+import org.opensearch.cluster.ClusterManagerMetrics;
 import org.opensearch.cluster.ClusterName;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.ClusterStateTaskConfig;
@@ -147,7 +148,7 @@ private ClusterManagerService createClusterManagerService(boolean makeClusterMan
                 .build(),
             new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
             threadPool,
-            metricsRegistry
+            new ClusterManagerMetrics(metricsRegistry)
         );
         final ClusterState initialClusterState = ClusterState.builder(new ClusterName(MasterServiceTests.class.getSimpleName()))
             .nodes(
@@ -221,7 +222,6 @@ public void onFailure(String source, Exception e) {
 
         nonClusterManager.close();
 
-        verify(metricsRegistry, times(2)).createHistogram(anyString(), anyString(), anyString());
         verify(clusterStateComputeHistogram, times(1)).record(anyDouble(), any());
     }
 
@@ -1100,7 +1100,7 @@ public void testLongClusterStateUpdateLogging() throws Exception {
                         .build(),
                     new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
                     threadPool,
-                    NoopMetricsRegistry.INSTANCE
+                    new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE)
                 )
             ) {
 
@@ -1296,7 +1296,7 @@ public void testAcking() throws InterruptedException {
                     .build(),
                 new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
                 threadPool,
-                metricsRegistry
+                new ClusterManagerMetrics(metricsRegistry)
             )
         ) {
 
@@ -1416,7 +1416,6 @@ public void onAckTimeout() {
             }
         }
 
-        verify(metricsRegistry, times(2)).createHistogram(anyString(), anyString(), anyString());
         verify(clusterStateComputeHistogram, times(2)).record(anyDouble(), any());
         verify(clusterStatePublishHistogram, times(1)).record(anyDouble());
     }
diff --git a/server/src/test/java/org/opensearch/cluster/service/TaskBatcherTests.java b/server/src/test/java/org/opensearch/cluster/service/TaskBatcherTests.java
index 070fde523fe87..b0916ce9236f7 100644
--- a/server/src/test/java/org/opensearch/cluster/service/TaskBatcherTests.java
+++ b/server/src/test/java/org/opensearch/cluster/service/TaskBatcherTests.java
@@ -218,7 +218,8 @@ public void testTasksAreExecutedInOrder() throws BrokenBarrierException, Interru
             executors[i] = new TaskExecutor();
         }
 
-        int tasksSubmittedPerThread = randomIntBetween(2, 1024);
+        // it will create at most 8192 threads, which will cause native memory oom. so we limit the number of created threads.
+        int tasksSubmittedPerThread = randomIntBetween(2, 128);
 
         CopyOnWriteArrayList<Tuple<String, Throwable>> failures = new CopyOnWriteArrayList<>();
         CountDownLatch updateLatch = new CountDownLatch(numberOfThreads * tasksSubmittedPerThread);
@@ -286,7 +287,7 @@ public void testNoTasksAreDroppedInParallelSubmission() throws BrokenBarrierExce
             executors[i] = new TaskExecutor();
         }
 
-        int tasksSubmittedPerThread = randomIntBetween(2, 1024);
+        int tasksSubmittedPerThread = randomIntBetween(2, 128);
 
         CopyOnWriteArrayList<Tuple<String, Throwable>> failures = new CopyOnWriteArrayList<>();
         CountDownLatch updateLatch = new CountDownLatch(numberOfThreads * tasksSubmittedPerThread);
diff --git a/server/src/test/java/org/opensearch/cluster/structure/RoutingIteratorTests.java b/server/src/test/java/org/opensearch/cluster/structure/RoutingIteratorTests.java
index 86adfc1279d9a..190ad3283dcfc 100644
--- a/server/src/test/java/org/opensearch/cluster/structure/RoutingIteratorTests.java
+++ b/server/src/test/java/org/opensearch/cluster/structure/RoutingIteratorTests.java
@@ -700,9 +700,18 @@ public void testWeightedRoutingWithDifferentWeights() {
                 .shard(0)
                 .activeInitializingShardsWeightedIt(weightedRouting, clusterState.nodes(), 1, false, null);
             assertEquals(1, shardIterator.size());
-            shardRouting = shardIterator.nextOrNull();
-            assertNotNull(shardRouting);
-            assertFalse(Arrays.asList("node2", "node1").contains(shardRouting.currentNodeId()));
+            assertEquals("node3", shardIterator.nextOrNull().currentNodeId());
+
+            weights = Map.of("zone1", -1.0, "zone2", 0.0, "zone3", 1.0);
+            weightedRouting = new WeightedRouting("zone", weights);
+            shardIterator = clusterState.routingTable()
+                .index("test")
+                .shard(0)
+                .activeInitializingShardsWeightedIt(weightedRouting, clusterState.nodes(), 1, true, null);
+            assertEquals(3, shardIterator.size());
+            assertEquals("node3", shardIterator.nextOrNull().currentNodeId());
+            assertNotEquals("node3", shardIterator.nextOrNull().currentNodeId());
+            assertNotEquals("node3", shardIterator.nextOrNull().currentNodeId());
 
             weights = Map.of("zone1", 3.0, "zone2", 2.0, "zone3", 0.0);
             weightedRouting = new WeightedRouting("zone", weights);
@@ -711,8 +720,138 @@ public void testWeightedRoutingWithDifferentWeights() {
                 .shard(0)
                 .activeInitializingShardsWeightedIt(weightedRouting, clusterState.nodes(), 1, true, null);
             assertEquals(3, shardIterator.size());
-            shardRouting = shardIterator.nextOrNull();
-            assertNotNull(shardRouting);
+            assertNotEquals("node3", shardIterator.nextOrNull().currentNodeId());
+            assertNotEquals("node3", shardIterator.nextOrNull().currentNodeId());
+            assertEquals("node3", shardIterator.nextOrNull().currentNodeId());
+        } finally {
+            terminate(threadPool);
+        }
+    }
+
+    public void testWeightedRoutingWithInitializingShards() {
+        TestThreadPool threadPool = null;
+        try {
+            Settings.Builder settings = Settings.builder()
+                .put("cluster.routing.allocation.node_concurrent_recoveries", 10)
+                .put("cluster.routing.allocation.awareness.attributes", "zone");
+            AllocationService strategy = createAllocationService(settings.build());
+
+            Metadata metadata = Metadata.builder()
+                .put(IndexMetadata.builder("test").settings(settings(Version.CURRENT)).numberOfShards(1).numberOfReplicas(2))
+                .build();
+
+            RoutingTable routingTable = RoutingTable.builder().addAsNew(metadata.index("test")).build();
+
+            ClusterState clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
+                .metadata(metadata)
+                .routingTable(routingTable)
+                .build();
+
+            threadPool = new TestThreadPool("testThatOnlyNodesSupport");
+            ClusterService clusterService = ClusterServiceUtils.createClusterService(threadPool);
+
+            Map<String, String> node1Attributes = new HashMap<>();
+            node1Attributes.put("zone", "zone1");
+            Map<String, String> node2Attributes = new HashMap<>();
+            node2Attributes.put("zone", "zone2");
+            Map<String, String> node3Attributes = new HashMap<>();
+            node3Attributes.put("zone", "zone3");
+
+            DiscoveryNodes nodes = DiscoveryNodes.builder()
+                .add(newNode("node1", unmodifiableMap(node1Attributes)))
+                .add(newNode("node2", unmodifiableMap(node2Attributes)))
+                .add(newNode("node3", unmodifiableMap(node3Attributes)))
+                .localNodeId("node1")
+                .build();
+            clusterState = ClusterState.builder(clusterState).nodes(nodes).build();
+            clusterState = strategy.reroute(clusterState, "reroute");
+
+            // Making the first shard as active
+            clusterState = startInitializingShardsAndReroute(strategy, clusterState);
+            // Making the second shard as active
+            clusterState = startRandomInitializingShard(clusterState, strategy);
+
+            String[] startedNodes = new String[2];
+            String[] startedZones = new String[2];
+            String initializingNode = null;
+            String initializingZone = null;
+            int i = 0;
+            for (ShardRouting shard : clusterState.routingTable().allShards()) {
+                if (shard.initializing()) {
+                    initializingNode = shard.currentNodeId();
+                    initializingZone = nodes.resolveNode(shard.currentNodeId()).getAttributes().get("zone");
+
+                } else {
+                    startedNodes[i] = shard.currentNodeId();
+                    startedZones[i++] = nodes.resolveNode(shard.currentNodeId()).getAttributes().get("zone");
+                }
+            }
+
+            Map<String, Double> weights = Map.of(startedZones[0], 1.0, initializingZone, 1.0, startedZones[1], 0.0);
+            WeightedRouting weightedRouting = new WeightedRouting("zone", weights);
+
+            // With fail open enabled set to false, we expect 2 shard routing, first one started, followed by initializing
+            ShardIterator shardIterator = clusterState.routingTable()
+                .index("test")
+                .shard(0)
+                .activeInitializingShardsWeightedIt(weightedRouting, clusterState.nodes(), 1, false, null);
+
+            assertEquals(2, shardIterator.size());
+            assertEquals(startedNodes[0], shardIterator.nextOrNull().currentNodeId());
+            assertEquals(initializingNode, shardIterator.nextOrNull().currentNodeId());
+
+            // With fail open enabled set to true, we expect 3 shard routing, first one started, followed by initializing, third one started
+            // with zero weight
+            shardIterator = clusterState.routingTable()
+                .index("test")
+                .shard(0)
+                .activeInitializingShardsWeightedIt(weightedRouting, clusterState.nodes(), 1, true, null);
+
+            assertEquals(3, shardIterator.size());
+            assertEquals(startedNodes[0], shardIterator.nextOrNull().currentNodeId());
+            assertEquals(initializingNode, shardIterator.nextOrNull().currentNodeId());
+            assertEquals(startedNodes[1], shardIterator.nextOrNull().currentNodeId());
+
+            weights = Map.of(initializingZone, 1.0, startedZones[0], 0.0, startedZones[1], 0.0);
+            weightedRouting = new WeightedRouting("zone", weights);
+
+            // only initializing shard has weight with fail open true
+            shardIterator = clusterState.routingTable()
+                .index("test")
+                .shard(0)
+                .activeInitializingShardsWeightedIt(weightedRouting, clusterState.nodes(), 1, false, null);
+            assertEquals(1, shardIterator.size());
+            assertEquals(initializingNode, shardIterator.nextOrNull().currentNodeId());
+
+            // only initializing shard has weight with fail open false
+            shardIterator = clusterState.routingTable()
+                .index("test")
+                .shard(0)
+                .activeInitializingShardsWeightedIt(weightedRouting, clusterState.nodes(), 1, true, null);
+            assertEquals(3, shardIterator.size());
+            assertEquals(initializingNode, shardIterator.nextOrNull().currentNodeId());
+            assertNotEquals(initializingNode, shardIterator.nextOrNull().currentNodeId());
+            assertNotEquals(initializingNode, shardIterator.nextOrNull().currentNodeId());
+
+            weights = Map.of(initializingZone, 0.0, startedZones[0], 1.0, startedZones[1], 0.0);
+            weightedRouting = new WeightedRouting("zone", weights);
+
+            shardIterator = clusterState.routingTable()
+                .index("test")
+                .shard(0)
+                .activeInitializingShardsWeightedIt(weightedRouting, clusterState.nodes(), 1, false, null);
+            assertEquals(1, shardIterator.size());
+            assertEquals(startedNodes[0], shardIterator.nextOrNull().currentNodeId());
+
+            shardIterator = clusterState.routingTable()
+                .index("test")
+                .shard(0)
+                .activeInitializingShardsWeightedIt(weightedRouting, clusterState.nodes(), 1, true, null);
+            assertEquals(3, shardIterator.size());
+            assertEquals(startedNodes[0], shardIterator.nextOrNull().currentNodeId());
+            assertEquals(startedNodes[1], shardIterator.nextOrNull().currentNodeId());
+            assertEquals(initializingNode, shardIterator.nextOrNull().currentNodeId());
+
         } finally {
             terminate(threadPool);
         }
diff --git a/server/src/test/java/org/opensearch/common/GlobTests.java b/server/src/test/java/org/opensearch/common/GlobTests.java
new file mode 100644
index 0000000000000..2bbe157be43cc
--- /dev/null
+++ b/server/src/test/java/org/opensearch/common/GlobTests.java
@@ -0,0 +1,67 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common;
+
+import org.opensearch.test.OpenSearchTestCase;
+
+public class GlobTests extends OpenSearchTestCase {
+
+    public void testGlobMatchForNull() {
+        assertFalse(Glob.globMatch(null, "test"));
+        assertFalse(Glob.globMatch("test", null));
+        assertFalse(Glob.globMatch(null, null));
+    }
+
+    public void testGlobMatchNoWildcard() {
+        assertTrue(Glob.globMatch("abcd", "abcd"));
+        assertFalse(Glob.globMatch("abcd", "foobar"));
+    }
+
+    public void testGlobMatchSingleWildcard() {
+        assertTrue(Glob.globMatch("*foo", "barfoo"));
+        assertFalse(Glob.globMatch("*foo", "foobar"));
+        assertTrue(Glob.globMatch("foo*", "foobarfoo"));
+        assertFalse(Glob.globMatch("foo*", "barfoobar"));
+        assertTrue(Glob.globMatch("foo*bar", "foobarnfoosbar"));
+    }
+
+    public void testGlobMatchMultipleWildcards() {
+        assertTrue(Glob.globMatch("*foo*", "barfoobar"));
+        assertFalse(Glob.globMatch("*foo*", "baroofbar"));
+        assertTrue(Glob.globMatch("*foo*bar", "abcdfooefghbar"));
+        assertFalse(Glob.globMatch("*foo*bar", "foonotbars"));
+    }
+
+    public void testGlobalMatchDoubleWildcard() {
+        assertTrue(Glob.globMatch("**foo", "barbarfoo"));
+        assertFalse(Glob.globMatch("**foo", "barbarfoowoof"));
+        assertTrue(Glob.globMatch("**bar**", "foobarfoo"));
+        assertFalse(Glob.globMatch("**bar**", "foobanfoo"));
+    }
+
+    public void testGlobMatchMultipleCharactersWithSingleWildcard() {
+        assertTrue(Glob.globMatch("a*b", "acb"));
+        assertTrue(Glob.globMatch("f*oo", "foo"));
+        assertTrue(Glob.globMatch("a*b", "aab"));
+        assertTrue(Glob.globMatch("a*b", "aaab"));
+    }
+
+    public void testGlobMatchWildcardWithEmptyString() {
+        assertTrue(Glob.globMatch("*", ""));
+        assertTrue(Glob.globMatch("a*", "a"));
+        assertFalse(Glob.globMatch("a*", ""));
+    }
+
+    public void testGlobMatchMultipleWildcardsWithMultipleCharacters() {
+        assertTrue(Glob.globMatch("a*b*c", "abc"));
+        assertTrue(Glob.globMatch("a*b*c", "axxxbxbc"));
+        assertFalse(Glob.globMatch("a*b*c", "abca"));
+        assertFalse(Glob.globMatch("a*b*c", "ac"));
+    }
+}
diff --git a/server/src/test/java/org/opensearch/common/blobstore/AsyncMultiStreamEncryptedBlobContainerTests.java b/server/src/test/java/org/opensearch/common/blobstore/AsyncMultiStreamEncryptedBlobContainerTests.java
index 1780819390052..aee4ae40d16a5 100644
--- a/server/src/test/java/org/opensearch/common/blobstore/AsyncMultiStreamEncryptedBlobContainerTests.java
+++ b/server/src/test/java/org/opensearch/common/blobstore/AsyncMultiStreamEncryptedBlobContainerTests.java
@@ -57,7 +57,7 @@ public void testReadBlobAsync() throws Exception {
         final ListenerTestUtils.CountingCompletionListener<ReadContext> completionListener =
             new ListenerTestUtils.CountingCompletionListener<>();
         final CompletableFuture<InputStreamContainer> streamContainerFuture = CompletableFuture.completedFuture(inputStreamContainer);
-        final ReadContext readContext = new ReadContext(size, List.of(() -> streamContainerFuture), null);
+        final ReadContext readContext = new ReadContext.Builder(size, List.of(() -> streamContainerFuture)).build();
 
         Mockito.doAnswer(invocation -> {
             ActionListener<ReadContext> readContextActionListener = invocation.getArgument(1);
@@ -103,7 +103,7 @@ public void testReadBlobAsyncException() throws Exception {
         final ListenerTestUtils.CountingCompletionListener<ReadContext> completionListener =
             new ListenerTestUtils.CountingCompletionListener<>();
         final CompletableFuture<InputStreamContainer> streamContainerFuture = CompletableFuture.completedFuture(inputStreamContainer);
-        final ReadContext readContext = new ReadContext(size, List.of(() -> streamContainerFuture), null);
+        final ReadContext readContext = new ReadContext.Builder(size, List.of(() -> streamContainerFuture)).build();
 
         Mockito.doAnswer(invocation -> {
             ActionListener<ReadContext> readContextActionListener = invocation.getArgument(1);
diff --git a/server/src/test/java/org/opensearch/common/blobstore/stream/read/listener/ReadContextListenerTests.java b/server/src/test/java/org/opensearch/common/blobstore/stream/read/listener/ReadContextListenerTests.java
index 0163c2275e7f4..c47874f3ba294 100644
--- a/server/src/test/java/org/opensearch/common/blobstore/stream/read/listener/ReadContextListenerTests.java
+++ b/server/src/test/java/org/opensearch/common/blobstore/stream/read/listener/ReadContextListenerTests.java
@@ -78,7 +78,7 @@ public void testReadContextListener() throws InterruptedException, IOException {
             UnaryOperator.identity(),
             MAX_CONCURRENT_STREAMS
         );
-        ReadContext readContext = new ReadContext((long) PART_SIZE * NUMBER_OF_PARTS, blobPartStreams, null);
+        ReadContext readContext = new ReadContext.Builder((long) PART_SIZE * NUMBER_OF_PARTS, blobPartStreams).build();
         readContextListener.onResponse(readContext);
 
         countDownLatch.await();
@@ -125,7 +125,7 @@ public int available() {
                 threadPool.generic()
             )
         );
-        ReadContext readContext = new ReadContext((long) (PART_SIZE + 1) * NUMBER_OF_PARTS, blobPartStreams, null);
+        ReadContext readContext = new ReadContext.Builder((long) (PART_SIZE + 1) * NUMBER_OF_PARTS, blobPartStreams).build();
         readContextListener.onResponse(readContext);
 
         countDownLatch.await();
@@ -178,7 +178,7 @@ public int read(byte[] b) throws IOException {
                 threadPool.generic()
             )
         );
-        ReadContext readContext = new ReadContext((long) (PART_SIZE + 1) * NUMBER_OF_PARTS + 1, blobPartStreams, null);
+        ReadContext readContext = new ReadContext.Builder((long) (PART_SIZE + 1) * NUMBER_OF_PARTS + 1, blobPartStreams).build();
         readContextListener.onResponse(readContext);
 
         countDownLatch.await();
@@ -203,7 +203,7 @@ public void testWriteToTempFile_alreadyExists_replacesFile() throws Exception {
             UnaryOperator.identity(),
             MAX_CONCURRENT_STREAMS
         );
-        ReadContext readContext = new ReadContext((long) (PART_SIZE + 1) * NUMBER_OF_PARTS, blobPartStreams, null);
+        ReadContext readContext = new ReadContext.Builder((long) (PART_SIZE + 1) * NUMBER_OF_PARTS, blobPartStreams).build();
         readContextListener.onResponse(readContext);
 
         countDownLatch.await();
diff --git a/server/src/test/java/org/opensearch/common/blobstore/transfer/RemoteTransferContainerTests.java b/server/src/test/java/org/opensearch/common/blobstore/transfer/RemoteTransferContainerTests.java
index 074f659850c7b..44f6a0e11251c 100644
--- a/server/src/test/java/org/opensearch/common/blobstore/transfer/RemoteTransferContainerTests.java
+++ b/server/src/test/java/org/opensearch/common/blobstore/transfer/RemoteTransferContainerTests.java
@@ -67,7 +67,7 @@ public OffsetRangeInputStream get(long size, long position) throws IOException {
                         return new OffsetRangeFileInputStream(testFile, size, position);
                     }
                 },
-                0,
+                0L,
                 false
             )
         ) {
@@ -89,7 +89,7 @@ public OffsetRangeInputStream get(long size, long position) throws IOException {
                         return new OffsetRangeFileInputStream(testFile, size, position);
                     }
                 },
-                0,
+                0L,
                 false
             )
         ) {
@@ -155,7 +155,7 @@ public OffsetRangeInputStream get(long size, long position) throws IOException {
                         return new OffsetRangeFileInputStream(testFile, size, position);
                     }
                 },
-                0,
+                0L,
                 false
             )
         ) {
@@ -223,7 +223,7 @@ public OffsetRangeInputStream get(long size, long position) throws IOException {
                         return new OffsetRangeFileInputStream(testFile, size, position);
                     }
                 },
-                0,
+                0L,
                 isRemoteDataIntegritySupported
             )
         ) {
@@ -286,7 +286,7 @@ public OffsetRangeInputStream get(long size, long position) throws IOException {
                         return new RateLimitingOffsetRangeInputStream(offsetRangeIndexInputStream, rateLimiterSupplier, null);
                     }
                 },
-                0,
+                0L,
                 true
             )
         ) {
@@ -347,7 +347,7 @@ public OffsetRangeInputStream get(long size, long position) throws IOException {
                         return new RateLimitingOffsetRangeInputStream(offsetRangeIndexInputStream, rateLimiterSupplier, null);
                     }
                 },
-                0,
+                0L,
                 true
             )
         ) {
diff --git a/server/src/test/java/org/opensearch/common/cache/serializer/ICacheKeySerializerTests.java b/server/src/test/java/org/opensearch/common/cache/serializer/ICacheKeySerializerTests.java
new file mode 100644
index 0000000000000..7713fdf1d0adc
--- /dev/null
+++ b/server/src/test/java/org/opensearch/common/cache/serializer/ICacheKeySerializerTests.java
@@ -0,0 +1,107 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.cache.serializer;
+
+import org.opensearch.OpenSearchException;
+import org.opensearch.common.Randomness;
+import org.opensearch.common.cache.ICacheKey;
+import org.opensearch.core.common.bytes.BytesArray;
+import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.UUID;
+
+public class ICacheKeySerializerTests extends OpenSearchTestCase {
+    // For these tests, we use BytesReference as K, since we already have a Serializer<BytesReference, byte[]> implementation
+    public void testEquality() throws Exception {
+        BytesReferenceSerializer keySer = new BytesReferenceSerializer();
+        ICacheKeySerializer<BytesReference> serializer = new ICacheKeySerializer<>(keySer);
+
+        int numDimensionsTested = 100;
+        for (int i = 0; i < numDimensionsTested; i++) {
+            String dim = getRandomDimValue();
+            ICacheKey<BytesReference> key = new ICacheKey<>(getRandomBytesReference(), List.of(dim));
+            byte[] serialized = serializer.serialize(key);
+            assertTrue(serializer.equals(key, serialized));
+            ICacheKey<BytesReference> deserialized = serializer.deserialize(serialized);
+            assertEquals(key, deserialized);
+            assertTrue(serializer.equals(deserialized, serialized));
+        }
+    }
+
+    public void testInvalidInput() throws Exception {
+        BytesReferenceSerializer keySer = new BytesReferenceSerializer();
+        ICacheKeySerializer<BytesReference> serializer = new ICacheKeySerializer<>(keySer);
+
+        Random rand = Randomness.get();
+        byte[] randomInput = new byte[1000];
+        rand.nextBytes(randomInput);
+
+        assertThrows(OpenSearchException.class, () -> serializer.deserialize(randomInput));
+    }
+
+    public void testDimNumbers() throws Exception {
+        BytesReferenceSerializer keySer = new BytesReferenceSerializer();
+        ICacheKeySerializer<BytesReference> serializer = new ICacheKeySerializer<>(keySer);
+
+        for (int numDims : new int[] { 0, 5, 1000 }) {
+            List<String> dims = new ArrayList<>();
+            for (int j = 0; j < numDims; j++) {
+                dims.add(getRandomDimValue());
+            }
+            ICacheKey<BytesReference> key = new ICacheKey<>(getRandomBytesReference(), dims);
+            byte[] serialized = serializer.serialize(key);
+            assertTrue(serializer.equals(key, serialized));
+            ICacheKey<BytesReference> deserialized = serializer.deserialize(serialized);
+            assertEquals(key, deserialized);
+        }
+    }
+
+    public void testHashCodes() throws Exception {
+        ICacheKey<String> key1 = new ICacheKey<>("key", List.of("dimension_value"));
+        ICacheKey<String> key2 = new ICacheKey<>("key", List.of("dimension_value"));
+
+        ICacheKey<String> key3 = new ICacheKey<>(null, List.of("dimension_value"));
+        ICacheKey<String> key4 = new ICacheKey<>(null, List.of("dimension_value"));
+
+        assertEquals(key1, key2);
+        assertEquals(key1.hashCode(), key2.hashCode());
+
+        assertEquals(key3, key4);
+        assertEquals(key3.hashCode(), key4.hashCode());
+
+        assertNotEquals(key1, key3);
+        assertNotEquals("string", key3);
+    }
+
+    public void testNullInputs() throws Exception {
+        BytesReferenceSerializer keySer = new BytesReferenceSerializer();
+        ICacheKeySerializer<BytesReference> serializer = new ICacheKeySerializer<>(keySer);
+
+        assertNull(serializer.deserialize(null));
+        ICacheKey<BytesReference> nullKey = new ICacheKey<>(null, List.of(getRandomDimValue()));
+        assertNull(serializer.serialize(nullKey));
+        assertNull(serializer.serialize(null));
+        assertNull(serializer.serialize(new ICacheKey<>(getRandomBytesReference(), null)));
+    }
+
+    private String getRandomDimValue() {
+        return UUID.randomUUID().toString();
+    }
+
+    private BytesReference getRandomBytesReference() {
+        byte[] bytesValue = new byte[1000];
+        Random rand = Randomness.get();
+        rand.nextBytes(bytesValue);
+        return new BytesArray(bytesValue);
+    }
+}
diff --git a/server/src/test/java/org/opensearch/common/cache/stats/DefaultCacheStatsHolderTests.java b/server/src/test/java/org/opensearch/common/cache/stats/DefaultCacheStatsHolderTests.java
new file mode 100644
index 0000000000000..c6e8252ddf806
--- /dev/null
+++ b/server/src/test/java/org/opensearch/common/cache/stats/DefaultCacheStatsHolderTests.java
@@ -0,0 +1,311 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.cache.stats;
+
+import org.opensearch.common.Randomness;
+import org.opensearch.common.metrics.CounterMetric;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+
+public class DefaultCacheStatsHolderTests extends OpenSearchTestCase {
+    private final String storeName = "dummy_store";
+
+    public void testAddAndGet() throws Exception {
+        List<String> dimensionNames = List.of("dim1", "dim2", "dim3", "dim4");
+        DefaultCacheStatsHolder cacheStatsHolder = new DefaultCacheStatsHolder(dimensionNames, storeName);
+        Map<String, List<String>> usedDimensionValues = DefaultCacheStatsHolderTests.getUsedDimensionValues(cacheStatsHolder, 10);
+        Map<List<String>, CacheStats> expected = DefaultCacheStatsHolderTests.populateStats(
+            cacheStatsHolder,
+            usedDimensionValues,
+            1000,
+            10
+        );
+
+        // test the value in the map is as expected for each distinct combination of values
+        for (List<String> dimensionValues : expected.keySet()) {
+            CacheStats expectedCounter = expected.get(dimensionValues);
+
+            ImmutableCacheStats actualStatsHolder = DefaultCacheStatsHolderTests.getNode(dimensionValues, cacheStatsHolder.getStatsRoot())
+                .getImmutableStats();
+            ImmutableCacheStats actualCacheStats = getNode(dimensionValues, cacheStatsHolder.getStatsRoot()).getImmutableStats();
+
+            assertEquals(expectedCounter.immutableSnapshot(), actualStatsHolder);
+            assertEquals(expectedCounter.immutableSnapshot(), actualCacheStats);
+        }
+
+        // Check overall total matches
+        CacheStats expectedTotal = new CacheStats();
+        for (List<String> dims : expected.keySet()) {
+            expectedTotal.add(expected.get(dims));
+        }
+        assertEquals(expectedTotal.immutableSnapshot(), cacheStatsHolder.getStatsRoot().getImmutableStats());
+
+        // Check sum of children stats are correct
+        assertSumOfChildrenStats(cacheStatsHolder.getStatsRoot());
+    }
+
+    public void testReset() throws Exception {
+        List<String> dimensionNames = List.of("dim1", "dim2");
+        DefaultCacheStatsHolder cacheStatsHolder = new DefaultCacheStatsHolder(dimensionNames, storeName);
+        Map<String, List<String>> usedDimensionValues = getUsedDimensionValues(cacheStatsHolder, 10);
+        Map<List<String>, CacheStats> expected = populateStats(cacheStatsHolder, usedDimensionValues, 100, 10);
+
+        cacheStatsHolder.reset();
+
+        for (List<String> dimensionValues : expected.keySet()) {
+            CacheStats originalCounter = expected.get(dimensionValues);
+            originalCounter.sizeInBytes = new CounterMetric();
+            originalCounter.items = new CounterMetric();
+
+            DefaultCacheStatsHolder.Node node = getNode(dimensionValues, cacheStatsHolder.getStatsRoot());
+            ImmutableCacheStats actual = node.getImmutableStats();
+            assertEquals(originalCounter.immutableSnapshot(), actual);
+        }
+    }
+
+    public void testDropStatsForDimensions() throws Exception {
+        List<String> dimensionNames = List.of("dim1", "dim2");
+        DefaultCacheStatsHolder cacheStatsHolder = new DefaultCacheStatsHolder(dimensionNames, storeName);
+
+        // Create stats for the following dimension sets
+        List<List<String>> populatedStats = List.of(List.of("A1", "B1"), List.of("A2", "B2"), List.of("A2", "B3"));
+        for (List<String> dims : populatedStats) {
+            cacheStatsHolder.incrementHits(dims);
+        }
+
+        assertEquals(3, cacheStatsHolder.getStatsRoot().getImmutableStats().getHits());
+
+        // When we invalidate A2, B2, we should lose the node for B2, but not B3 or A2.
+
+        cacheStatsHolder.removeDimensions(List.of("A2", "B2"));
+
+        assertEquals(2, cacheStatsHolder.getStatsRoot().getImmutableStats().getHits());
+        assertNull(getNode(List.of("A2", "B2"), cacheStatsHolder.getStatsRoot()));
+        assertNotNull(getNode(List.of("A2"), cacheStatsHolder.getStatsRoot()));
+        assertNotNull(getNode(List.of("A2", "B3"), cacheStatsHolder.getStatsRoot()));
+
+        // When we invalidate A1, B1, we should lose the nodes for B1 and also A1, as it has no more children.
+
+        cacheStatsHolder.removeDimensions(List.of("A1", "B1"));
+
+        assertEquals(1, cacheStatsHolder.getStatsRoot().getImmutableStats().getHits());
+        assertNull(getNode(List.of("A1", "B1"), cacheStatsHolder.getStatsRoot()));
+        assertNull(getNode(List.of("A1"), cacheStatsHolder.getStatsRoot()));
+
+        // When we invalidate the last node, all nodes should be deleted except the root node
+
+        cacheStatsHolder.removeDimensions(List.of("A2", "B3"));
+        assertEquals(0, cacheStatsHolder.getStatsRoot().getImmutableStats().getHits());
+        assertEquals(0, cacheStatsHolder.getStatsRoot().children.size());
+    }
+
+    public void testCount() throws Exception {
+        List<String> dimensionNames = List.of("dim1", "dim2");
+        DefaultCacheStatsHolder cacheStatsHolder = new DefaultCacheStatsHolder(dimensionNames, storeName);
+        Map<String, List<String>> usedDimensionValues = getUsedDimensionValues(cacheStatsHolder, 10);
+        Map<List<String>, CacheStats> expected = populateStats(cacheStatsHolder, usedDimensionValues, 100, 10);
+
+        long expectedCount = 0L;
+        for (CacheStats counter : expected.values()) {
+            expectedCount += counter.getItems();
+        }
+        assertEquals(expectedCount, cacheStatsHolder.count());
+    }
+
+    public void testConcurrentRemoval() throws Exception {
+        List<String> dimensionNames = List.of("dim1", "dim2");
+        DefaultCacheStatsHolder cacheStatsHolder = new DefaultCacheStatsHolder(dimensionNames, storeName);
+
+        // Create stats for the following dimension sets
+        List<List<String>> populatedStats = List.of(List.of("A1", "B1"), List.of("A2", "B2"), List.of("A2", "B3"));
+        for (List<String> dims : populatedStats) {
+            cacheStatsHolder.incrementHits(dims);
+        }
+
+        // Remove (A2, B2) and (A1, B1), before re-adding (A2, B2). At the end we should have stats for (A2, B2) but not (A1, B1).
+
+        Thread[] threads = new Thread[3];
+        CountDownLatch countDownLatch = new CountDownLatch(3);
+        threads[0] = new Thread(() -> {
+            cacheStatsHolder.removeDimensions(List.of("A2", "B2"));
+            countDownLatch.countDown();
+        });
+        threads[1] = new Thread(() -> {
+            cacheStatsHolder.removeDimensions(List.of("A1", "B1"));
+            countDownLatch.countDown();
+        });
+        threads[2] = new Thread(() -> {
+            cacheStatsHolder.incrementMisses(List.of("A2", "B2"));
+            cacheStatsHolder.incrementMisses(List.of("A2", "B3"));
+            countDownLatch.countDown();
+        });
+        for (Thread thread : threads) {
+            thread.start();
+            // Add short sleep to ensure threads start their functions in order (so that incrementing doesn't happen before removal)
+            Thread.sleep(1);
+        }
+        countDownLatch.await();
+        assertNull(getNode(List.of("A1", "B1"), cacheStatsHolder.getStatsRoot()));
+        assertNull(getNode(List.of("A1"), cacheStatsHolder.getStatsRoot()));
+        assertNotNull(getNode(List.of("A2", "B2"), cacheStatsHolder.getStatsRoot()));
+        assertEquals(
+            new ImmutableCacheStats(0, 1, 0, 0, 0),
+            getNode(List.of("A2", "B2"), cacheStatsHolder.getStatsRoot()).getImmutableStats()
+        );
+        assertEquals(
+            new ImmutableCacheStats(1, 1, 0, 0, 0),
+            getNode(List.of("A2", "B3"), cacheStatsHolder.getStatsRoot()).getImmutableStats()
+        );
+    }
+
+    /**
+     * Returns the node found by following these dimension values down from the root node.
+     * Returns null if no such node exists.
+     */
+    static DefaultCacheStatsHolder.Node getNode(List<String> dimensionValues, DefaultCacheStatsHolder.Node root) {
+        DefaultCacheStatsHolder.Node current = root;
+        for (String dimensionValue : dimensionValues) {
+            current = current.getChildren().get(dimensionValue);
+            if (current == null) {
+                return null;
+            }
+        }
+        return current;
+    }
+
+    static Map<List<String>, CacheStats> populateStats(
+        DefaultCacheStatsHolder cacheStatsHolder,
+        Map<String, List<String>> usedDimensionValues,
+        int numDistinctValuePairs,
+        int numRepetitionsPerValue
+    ) throws InterruptedException {
+        return populateStats(List.of(cacheStatsHolder), usedDimensionValues, numDistinctValuePairs, numRepetitionsPerValue);
+    }
+
+    static Map<List<String>, CacheStats> populateStats(
+        List<DefaultCacheStatsHolder> cacheStatsHolders,
+        Map<String, List<String>> usedDimensionValues,
+        int numDistinctValuePairs,
+        int numRepetitionsPerValue
+    ) throws InterruptedException {
+        for (DefaultCacheStatsHolder statsHolder : cacheStatsHolders) {
+            assertEquals(cacheStatsHolders.get(0).getDimensionNames(), statsHolder.getDimensionNames());
+        }
+        Map<List<String>, CacheStats> expected = new ConcurrentHashMap<>();
+        Thread[] threads = new Thread[numDistinctValuePairs];
+        CountDownLatch countDownLatch = new CountDownLatch(numDistinctValuePairs);
+        Random rand = Randomness.get();
+        List<List<String>> dimensionsForThreads = new ArrayList<>();
+        for (int i = 0; i < numDistinctValuePairs; i++) {
+            dimensionsForThreads.add(getRandomDimList(cacheStatsHolders.get(0).getDimensionNames(), usedDimensionValues, true, rand));
+            int finalI = i;
+            threads[i] = new Thread(() -> {
+                Random threadRand = Randomness.get();
+                List<String> dimensions = dimensionsForThreads.get(finalI);
+                expected.computeIfAbsent(dimensions, (key) -> new CacheStats());
+                for (DefaultCacheStatsHolder cacheStatsHolder : cacheStatsHolders) {
+                    for (int j = 0; j < numRepetitionsPerValue; j++) {
+                        CacheStats statsToInc = new CacheStats(
+                            threadRand.nextInt(10),
+                            threadRand.nextInt(10),
+                            threadRand.nextInt(10),
+                            threadRand.nextInt(5000),
+                            threadRand.nextInt(10)
+                        );
+                        expected.get(dimensions).hits.inc(statsToInc.getHits());
+                        expected.get(dimensions).misses.inc(statsToInc.getMisses());
+                        expected.get(dimensions).evictions.inc(statsToInc.getEvictions());
+                        expected.get(dimensions).sizeInBytes.inc(statsToInc.getSizeInBytes());
+                        expected.get(dimensions).items.inc(statsToInc.getItems());
+                        DefaultCacheStatsHolderTests.populateStatsHolderFromStatsValueMap(cacheStatsHolder, Map.of(dimensions, statsToInc));
+                    }
+                }
+                countDownLatch.countDown();
+            });
+        }
+        for (Thread thread : threads) {
+            thread.start();
+        }
+        countDownLatch.await();
+        return expected;
+    }
+
+    private static List<String> getRandomDimList(
+        List<String> dimensionNames,
+        Map<String, List<String>> usedDimensionValues,
+        boolean pickValueForAllDims,
+        Random rand
+    ) {
+        List<String> result = new ArrayList<>();
+        for (String dimName : dimensionNames) {
+            if (pickValueForAllDims || rand.nextBoolean()) { // if pickValueForAllDims, always pick a value for each dimension, otherwise do
+                // so 50% of the time
+                int index = between(0, usedDimensionValues.get(dimName).size() - 1);
+                result.add(usedDimensionValues.get(dimName).get(index));
+            }
+        }
+        return result;
+    }
+
+    static Map<String, List<String>> getUsedDimensionValues(DefaultCacheStatsHolder cacheStatsHolder, int numValuesPerDim) {
+        Map<String, List<String>> usedDimensionValues = new HashMap<>();
+        for (int i = 0; i < cacheStatsHolder.getDimensionNames().size(); i++) {
+            List<String> values = new ArrayList<>();
+            for (int j = 0; j < numValuesPerDim; j++) {
+                values.add(UUID.randomUUID().toString());
+            }
+            usedDimensionValues.put(cacheStatsHolder.getDimensionNames().get(i), values);
+        }
+        return usedDimensionValues;
+    }
+
+    private void assertSumOfChildrenStats(DefaultCacheStatsHolder.Node current) {
+        if (!current.children.isEmpty()) {
+            CacheStats expectedTotal = new CacheStats();
+            for (DefaultCacheStatsHolder.Node child : current.children.values()) {
+                expectedTotal.add(child.getImmutableStats());
+            }
+            assertEquals(expectedTotal.immutableSnapshot(), current.getImmutableStats());
+            for (DefaultCacheStatsHolder.Node child : current.children.values()) {
+                assertSumOfChildrenStats(child);
+            }
+        }
+    }
+
+    public static void populateStatsHolderFromStatsValueMap(
+        DefaultCacheStatsHolder cacheStatsHolder,
+        Map<List<String>, CacheStats> statsMap
+    ) {
+        for (Map.Entry<List<String>, CacheStats> entry : statsMap.entrySet()) {
+            CacheStats stats = entry.getValue();
+            List<String> dims = entry.getKey();
+            for (int i = 0; i < stats.getHits(); i++) {
+                cacheStatsHolder.incrementHits(dims);
+            }
+            for (int i = 0; i < stats.getMisses(); i++) {
+                cacheStatsHolder.incrementMisses(dims);
+            }
+            for (int i = 0; i < stats.getEvictions(); i++) {
+                cacheStatsHolder.incrementEvictions(dims);
+            }
+            cacheStatsHolder.incrementSizeInBytes(dims, stats.getSizeInBytes());
+            for (int i = 0; i < stats.getItems(); i++) {
+                cacheStatsHolder.incrementItems(dims);
+            }
+        }
+    }
+}
diff --git a/server/src/test/java/org/opensearch/common/cache/stats/ImmutableCacheStatsHolderTests.java b/server/src/test/java/org/opensearch/common/cache/stats/ImmutableCacheStatsHolderTests.java
new file mode 100644
index 0000000000000..285840a3451c6
--- /dev/null
+++ b/server/src/test/java/org/opensearch/common/cache/stats/ImmutableCacheStatsHolderTests.java
@@ -0,0 +1,370 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.cache.stats;
+
+import org.opensearch.common.io.stream.BytesStreamOutput;
+import org.opensearch.common.xcontent.XContentFactory;
+import org.opensearch.common.xcontent.XContentHelper;
+import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.core.common.io.stream.BytesStreamInput;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
+import org.opensearch.core.xcontent.ToXContent;
+import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.function.BiConsumer;
+
+public class ImmutableCacheStatsHolderTests extends OpenSearchTestCase {
+    private final String storeName = "dummy_store";
+
+    public void testSerialization() throws Exception {
+        List<String> dimensionNames = List.of("dim1", "dim2", "dim3");
+        String[] levels = dimensionNames.toArray(new String[0]);
+        DefaultCacheStatsHolder statsHolder = new DefaultCacheStatsHolder(dimensionNames, storeName);
+        Map<String, List<String>> usedDimensionValues = DefaultCacheStatsHolderTests.getUsedDimensionValues(statsHolder, 10);
+        DefaultCacheStatsHolderTests.populateStats(statsHolder, usedDimensionValues, 100, 10);
+        ImmutableCacheStatsHolder stats = statsHolder.getImmutableCacheStatsHolder(levels);
+        assertNotEquals(0, stats.getStatsRoot().children.size());
+
+        BytesStreamOutput os = new BytesStreamOutput();
+        stats.writeTo(os);
+        BytesStreamInput is = new BytesStreamInput(BytesReference.toBytes(os.bytes()));
+        ImmutableCacheStatsHolder deserialized = new ImmutableCacheStatsHolder(is);
+
+        assertEquals(stats, deserialized);
+
+        // also test empty dimension stats
+        ImmutableCacheStatsHolder emptyDims = statsHolder.getImmutableCacheStatsHolder(new String[] {});
+        assertEquals(0, emptyDims.getStatsRoot().children.size());
+        assertEquals(stats.getTotalStats(), emptyDims.getTotalStats());
+
+        os = new BytesStreamOutput();
+        emptyDims.writeTo(os);
+        is = new BytesStreamInput(BytesReference.toBytes(os.bytes()));
+        deserialized = new ImmutableCacheStatsHolder(is);
+
+        assertEquals(emptyDims, deserialized);
+    }
+
+    public void testEquals() throws Exception {
+        List<String> dimensionNames = List.of("dim1", "dim2", "dim3");
+        String[] levels = dimensionNames.toArray(new String[0]);
+        DefaultCacheStatsHolder statsHolder = new DefaultCacheStatsHolder(dimensionNames, storeName);
+        DefaultCacheStatsHolder differentStoreNameStatsHolder = new DefaultCacheStatsHolder(dimensionNames, "nonMatchingStoreName");
+        DefaultCacheStatsHolder nonMatchingStatsHolder = new DefaultCacheStatsHolder(dimensionNames, storeName);
+        Map<String, List<String>> usedDimensionValues = DefaultCacheStatsHolderTests.getUsedDimensionValues(statsHolder, 10);
+        DefaultCacheStatsHolderTests.populateStats(List.of(statsHolder, differentStoreNameStatsHolder), usedDimensionValues, 100, 10);
+        DefaultCacheStatsHolderTests.populateStats(nonMatchingStatsHolder, usedDimensionValues, 100, 10);
+        ImmutableCacheStatsHolder stats = statsHolder.getImmutableCacheStatsHolder(levels);
+
+        ImmutableCacheStatsHolder secondStats = statsHolder.getImmutableCacheStatsHolder(levels);
+        assertEquals(stats, secondStats);
+        ImmutableCacheStatsHolder nonMatchingStats = nonMatchingStatsHolder.getImmutableCacheStatsHolder(levels);
+        assertNotEquals(stats, nonMatchingStats);
+        ImmutableCacheStatsHolder differentStoreNameStats = differentStoreNameStatsHolder.getImmutableCacheStatsHolder(levels);
+        assertNotEquals(stats, differentStoreNameStats);
+    }
+
+    public void testGet() throws Exception {
+        List<String> dimensionNames = List.of("dim1", "dim2", "dim3", "dim4");
+        DefaultCacheStatsHolder cacheStatsHolder = new DefaultCacheStatsHolder(dimensionNames, storeName);
+        Map<String, List<String>> usedDimensionValues = DefaultCacheStatsHolderTests.getUsedDimensionValues(cacheStatsHolder, 10);
+        Map<List<String>, CacheStats> expected = DefaultCacheStatsHolderTests.populateStats(
+            cacheStatsHolder,
+            usedDimensionValues,
+            1000,
+            10
+        );
+        ImmutableCacheStatsHolder stats = cacheStatsHolder.getImmutableCacheStatsHolder(dimensionNames.toArray(new String[0]));
+
+        // test the value in the map is as expected for each distinct combination of values
+        for (List<String> dimensionValues : expected.keySet()) {
+            CacheStats expectedCounter = expected.get(dimensionValues);
+
+            ImmutableCacheStats actualCacheStatsHolder = DefaultCacheStatsHolderTests.getNode(
+                dimensionValues,
+                cacheStatsHolder.getStatsRoot()
+            ).getImmutableStats();
+            ImmutableCacheStats actualImmutableCacheStatsHolder = getNode(dimensionValues, stats.getStatsRoot()).getStats();
+
+            assertEquals(expectedCounter.immutableSnapshot(), actualCacheStatsHolder);
+            assertEquals(expectedCounter.immutableSnapshot(), actualImmutableCacheStatsHolder);
+        }
+
+        // test gets for total (this also checks sum-of-children logic)
+        CacheStats expectedTotal = new CacheStats();
+        for (List<String> dims : expected.keySet()) {
+            expectedTotal.add(expected.get(dims));
+        }
+        assertEquals(expectedTotal.immutableSnapshot(), stats.getTotalStats());
+
+        assertEquals(expectedTotal.getHits(), stats.getTotalHits());
+        assertEquals(expectedTotal.getMisses(), stats.getTotalMisses());
+        assertEquals(expectedTotal.getEvictions(), stats.getTotalEvictions());
+        assertEquals(expectedTotal.getSizeInBytes(), stats.getTotalSizeInBytes());
+        assertEquals(expectedTotal.getItems(), stats.getTotalItems());
+
+        assertSumOfChildrenStats(stats.getStatsRoot());
+    }
+
+    public void testEmptyDimsList() throws Exception {
+        // If the dimension list is empty, the tree should have only the root node containing the total stats.
+        DefaultCacheStatsHolder cacheStatsHolder = new DefaultCacheStatsHolder(List.of(), storeName);
+        Map<String, List<String>> usedDimensionValues = DefaultCacheStatsHolderTests.getUsedDimensionValues(cacheStatsHolder, 100);
+        DefaultCacheStatsHolderTests.populateStats(cacheStatsHolder, usedDimensionValues, 10, 100);
+        ImmutableCacheStatsHolder stats = cacheStatsHolder.getImmutableCacheStatsHolder(null);
+
+        ImmutableCacheStatsHolder.Node statsRoot = stats.getStatsRoot();
+        assertEquals(0, statsRoot.children.size());
+        assertEquals(stats.getTotalStats(), statsRoot.getStats());
+    }
+
+    public void testAggregateByAllDimensions() throws Exception {
+        // Aggregating with all dimensions as levels should just give us the same values that were in the original map
+        List<String> dimensionNames = List.of("dim1", "dim2", "dim3", "dim4");
+        DefaultCacheStatsHolder statsHolder = new DefaultCacheStatsHolder(dimensionNames, storeName);
+        Map<String, List<String>> usedDimensionValues = DefaultCacheStatsHolderTests.getUsedDimensionValues(statsHolder, 10);
+        Map<List<String>, CacheStats> expected = DefaultCacheStatsHolderTests.populateStats(statsHolder, usedDimensionValues, 1000, 10);
+        ImmutableCacheStatsHolder stats = statsHolder.getImmutableCacheStatsHolder(dimensionNames.toArray(new String[0]));
+
+        for (Map.Entry<List<String>, CacheStats> expectedEntry : expected.entrySet()) {
+            List<String> dimensionValues = new ArrayList<>();
+            for (String dimValue : expectedEntry.getKey()) {
+                dimensionValues.add(dimValue);
+            }
+            assertEquals(expectedEntry.getValue().immutableSnapshot(), getNode(dimensionValues, stats.statsRoot).getStats());
+        }
+        assertSumOfChildrenStats(stats.statsRoot);
+    }
+
+    public void testAggregateBySomeDimensions() throws Exception {
+        List<String> dimensionNames = List.of("dim1", "dim2", "dim3", "dim4");
+        DefaultCacheStatsHolder statsHolder = new DefaultCacheStatsHolder(dimensionNames, storeName);
+        Map<String, List<String>> usedDimensionValues = DefaultCacheStatsHolderTests.getUsedDimensionValues(statsHolder, 10);
+        Map<List<String>, CacheStats> expected = DefaultCacheStatsHolderTests.populateStats(statsHolder, usedDimensionValues, 1000, 10);
+
+        for (int i = 0; i < (1 << dimensionNames.size()); i++) {
+            // Test each combination of possible levels
+            List<String> levels = new ArrayList<>();
+            for (int nameIndex = 0; nameIndex < dimensionNames.size(); nameIndex++) {
+                if ((i & (1 << nameIndex)) != 0) {
+                    levels.add(dimensionNames.get(nameIndex));
+                }
+            }
+
+            if (levels.size() == 0) {
+                // If we pass empty levels to CacheStatsHolder to aggregate by, we should only get a root node with the total stats in it
+                ImmutableCacheStatsHolder stats = statsHolder.getImmutableCacheStatsHolder(levels.toArray(new String[0]));
+                assertEquals(statsHolder.getStatsRoot().getImmutableStats(), stats.getStatsRoot().getStats());
+                assertEquals(0, stats.getStatsRoot().children.size());
+            } else {
+                ImmutableCacheStatsHolder stats = statsHolder.getImmutableCacheStatsHolder(levels.toArray(new String[0]));
+                Map<List<String>, ImmutableCacheStatsHolder.Node> aggregatedLeafNodes = getAllLeafNodes(stats.statsRoot);
+
+                for (Map.Entry<List<String>, ImmutableCacheStatsHolder.Node> aggEntry : aggregatedLeafNodes.entrySet()) {
+                    CacheStats expectedCounter = new CacheStats();
+                    for (List<String> expectedDims : expected.keySet()) {
+                        if (expectedDims.containsAll(aggEntry.getKey())) {
+                            expectedCounter.add(expected.get(expectedDims));
+                        }
+                    }
+                    assertEquals(expectedCounter.immutableSnapshot(), aggEntry.getValue().getStats());
+                }
+                assertSumOfChildrenStats(stats.statsRoot);
+            }
+        }
+    }
+
+    public void testXContentForLevels() throws Exception {
+        List<String> dimensionNames = List.of("A", "B", "C");
+
+        DefaultCacheStatsHolder statsHolder = new DefaultCacheStatsHolder(dimensionNames, storeName);
+        DefaultCacheStatsHolderTests.populateStatsHolderFromStatsValueMap(
+            statsHolder,
+            Map.of(
+                List.of("A1", "B1", "C1"),
+                new CacheStats(1, 1, 1, 1, 1),
+                List.of("A1", "B1", "C2"),
+                new CacheStats(2, 2, 2, 2, 2),
+                List.of("A1", "B2", "C1"),
+                new CacheStats(3, 3, 3, 3, 3),
+                List.of("A2", "B1", "C3"),
+                new CacheStats(4, 4, 4, 4, 4)
+            )
+        );
+        ImmutableCacheStatsHolder stats = statsHolder.getImmutableCacheStatsHolder(dimensionNames.toArray(new String[0]));
+
+        XContentBuilder builder = XContentFactory.jsonBuilder();
+        ToXContent.Params params = ToXContent.EMPTY_PARAMS;
+
+        builder.startObject();
+        stats.toXContent(builder, params);
+        builder.endObject();
+        String resultString = builder.toString();
+        Map<String, Object> result = XContentHelper.convertToMap(MediaTypeRegistry.JSON.xContent(), resultString, true);
+
+        Map<String, BiConsumer<CacheStats, Integer>> fieldNamesMap = Map.of(
+            ImmutableCacheStats.Fields.SIZE_IN_BYTES,
+            (counter, value) -> counter.sizeInBytes.inc(value),
+            ImmutableCacheStats.Fields.EVICTIONS,
+            (counter, value) -> counter.evictions.inc(value),
+            ImmutableCacheStats.Fields.HIT_COUNT,
+            (counter, value) -> counter.hits.inc(value),
+            ImmutableCacheStats.Fields.MISS_COUNT,
+            (counter, value) -> counter.misses.inc(value),
+            ImmutableCacheStats.Fields.ITEM_COUNT,
+            (counter, value) -> counter.items.inc(value)
+        );
+
+        Map<List<String>, ImmutableCacheStatsHolder.Node> leafNodes = getAllLeafNodes(stats.getStatsRoot());
+        for (Map.Entry<List<String>, ImmutableCacheStatsHolder.Node> entry : leafNodes.entrySet()) {
+            List<String> xContentKeys = new ArrayList<>();
+            for (int i = 0; i < dimensionNames.size(); i++) {
+                xContentKeys.add(dimensionNames.get(i));
+                xContentKeys.add(entry.getKey().get(i));
+            }
+            CacheStats counterFromXContent = new CacheStats();
+
+            for (Map.Entry<String, BiConsumer<CacheStats, Integer>> fieldNamesEntry : fieldNamesMap.entrySet()) {
+                List<String> fullXContentKeys = new ArrayList<>(xContentKeys);
+                fullXContentKeys.add(fieldNamesEntry.getKey());
+                int valueInXContent = (int) getValueFromNestedXContentMap(result, fullXContentKeys);
+                BiConsumer<CacheStats, Integer> incrementer = fieldNamesEntry.getValue();
+                incrementer.accept(counterFromXContent, valueInXContent);
+            }
+
+            ImmutableCacheStats expected = entry.getValue().getStats();
+            assertEquals(counterFromXContent.immutableSnapshot(), expected);
+        }
+    }
+
+    public void testXContent() throws Exception {
+        // Tests logic of filtering levels out, logic for aggregating by those levels is already covered
+        List<String> dimensionNames = List.of("A", "B", "C");
+        DefaultCacheStatsHolder statsHolder = new DefaultCacheStatsHolder(dimensionNames, storeName);
+        Map<String, List<String>> usedDimensionValues = DefaultCacheStatsHolderTests.getUsedDimensionValues(statsHolder, 10);
+        DefaultCacheStatsHolderTests.populateStats(statsHolder, usedDimensionValues, 100, 10);
+
+        // If the levels in the params are empty or contains only unrecognized levels, we should only see the total stats and no level
+        // aggregation
+        List<List<String>> levelsList = List.of(List.of(), List.of("D"));
+        for (List<String> levels : levelsList) {
+            ImmutableCacheStatsHolder stats = statsHolder.getImmutableCacheStatsHolder(levels.toArray(new String[0]));
+            ToXContent.Params params = getLevelParams(levels);
+            XContentBuilder builder = XContentFactory.jsonBuilder();
+            builder.startObject();
+            stats.toXContent(builder, params);
+            builder.endObject();
+
+            String resultString = builder.toString();
+            Map<String, Object> result = XContentHelper.convertToMap(MediaTypeRegistry.JSON.xContent(), resultString, true);
+
+            assertTotalStatsPresentInXContentResponse(result);
+            // assert there are no other entries in the map besides these 6
+            assertEquals(6, result.size());
+        }
+
+        // if we pass recognized levels in any order, alongside ignored unrecognized levels, we should see the above plus level aggregation
+        List<String> levels = List.of("C", "A", "E");
+        ImmutableCacheStatsHolder stats = statsHolder.getImmutableCacheStatsHolder(levels.toArray(new String[0]));
+        ToXContent.Params params = getLevelParams(levels);
+        XContentBuilder builder = XContentFactory.jsonBuilder();
+        builder.startObject();
+        stats.toXContent(builder, params);
+        builder.endObject();
+
+        String resultString = builder.toString();
+        Map<String, Object> result = XContentHelper.convertToMap(MediaTypeRegistry.JSON.xContent(), resultString, true);
+        assertTotalStatsPresentInXContentResponse(result);
+        assertNotNull(result.get("A"));
+        assertEquals(7, result.size());
+    }
+
+    private void assertTotalStatsPresentInXContentResponse(Map<String, Object> result) {
+        // assert the total stats are present
+        assertNotEquals(0, (int) result.get(ImmutableCacheStats.Fields.SIZE_IN_BYTES));
+        assertNotEquals(0, (int) result.get(ImmutableCacheStats.Fields.EVICTIONS));
+        assertNotEquals(0, (int) result.get(ImmutableCacheStats.Fields.HIT_COUNT));
+        assertNotEquals(0, (int) result.get(ImmutableCacheStats.Fields.MISS_COUNT));
+        assertNotEquals(0, (int) result.get(ImmutableCacheStats.Fields.ITEM_COUNT));
+        // assert the store name is present
+        assertEquals(storeName, (String) result.get(ImmutableCacheStatsHolder.STORE_NAME_FIELD));
+    }
+
+    private ToXContent.Params getLevelParams(List<String> levels) {
+        Map<String, String> paramMap = new HashMap<>();
+        if (!levels.isEmpty()) {
+            paramMap.put("level", String.join(",", levels));
+        }
+        return new ToXContent.MapParams(paramMap);
+    }
+
+    public static Object getValueFromNestedXContentMap(Map<String, Object> xContentMap, List<String> keys) {
+        Map<String, Object> current = xContentMap;
+        for (int i = 0; i < keys.size() - 1; i++) {
+            Object next = current.get(keys.get(i));
+            if (next == null) {
+                return null;
+            }
+            current = (Map<String, Object>) next;
+        }
+        return current.get(keys.get(keys.size() - 1));
+    }
+
+    // Get a map from the list of dimension values to the corresponding leaf node.
+    private Map<List<String>, ImmutableCacheStatsHolder.Node> getAllLeafNodes(ImmutableCacheStatsHolder.Node root) {
+        Map<List<String>, ImmutableCacheStatsHolder.Node> result = new HashMap<>();
+        getAllLeafNodesHelper(result, root, new ArrayList<>());
+        return result;
+    }
+
+    private void getAllLeafNodesHelper(
+        Map<List<String>, ImmutableCacheStatsHolder.Node> result,
+        ImmutableCacheStatsHolder.Node current,
+        List<String> pathToCurrent
+    ) {
+        if (current.children.isEmpty()) {
+            result.put(pathToCurrent, current);
+        } else {
+            for (Map.Entry<String, ImmutableCacheStatsHolder.Node> entry : current.children.entrySet()) {
+                List<String> newPath = new ArrayList<>(pathToCurrent);
+                newPath.add(entry.getKey());
+                getAllLeafNodesHelper(result, entry.getValue(), newPath);
+            }
+        }
+    }
+
+    private ImmutableCacheStatsHolder.Node getNode(List<String> dimensionValues, ImmutableCacheStatsHolder.Node root) {
+        ImmutableCacheStatsHolder.Node current = root;
+        for (String dimensionValue : dimensionValues) {
+            current = current.getChildren().get(dimensionValue);
+            if (current == null) {
+                return null;
+            }
+        }
+        return current;
+    }
+
+    private void assertSumOfChildrenStats(ImmutableCacheStatsHolder.Node current) {
+        if (!current.children.isEmpty()) {
+            CacheStats expectedTotal = new CacheStats();
+            for (ImmutableCacheStatsHolder.Node child : current.children.values()) {
+                expectedTotal.add(child.getStats());
+            }
+            assertEquals(expectedTotal.immutableSnapshot(), current.getStats());
+            for (ImmutableCacheStatsHolder.Node child : current.children.values()) {
+                assertSumOfChildrenStats(child);
+            }
+        }
+    }
+}
diff --git a/server/src/test/java/org/opensearch/common/cache/stats/ImmutableCacheStatsTests.java b/server/src/test/java/org/opensearch/common/cache/stats/ImmutableCacheStatsTests.java
new file mode 100644
index 0000000000000..50ddd81943c3b
--- /dev/null
+++ b/server/src/test/java/org/opensearch/common/cache/stats/ImmutableCacheStatsTests.java
@@ -0,0 +1,47 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.cache.stats;
+
+import org.opensearch.common.io.stream.BytesStreamOutput;
+import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.core.common.io.stream.BytesStreamInput;
+import org.opensearch.test.OpenSearchTestCase;
+
+public class ImmutableCacheStatsTests extends OpenSearchTestCase {
+    public void testSerialization() throws Exception {
+        ImmutableCacheStats immutableCacheStats = new ImmutableCacheStats(1, 2, 3, 4, 5);
+        BytesStreamOutput os = new BytesStreamOutput();
+        immutableCacheStats.writeTo(os);
+        BytesStreamInput is = new BytesStreamInput(BytesReference.toBytes(os.bytes()));
+        ImmutableCacheStats deserialized = new ImmutableCacheStats(is);
+
+        assertEquals(immutableCacheStats, deserialized);
+    }
+
+    public void testAddSnapshots() throws Exception {
+        ImmutableCacheStats ics1 = new ImmutableCacheStats(1, 2, 3, 4, 5);
+        ImmutableCacheStats ics2 = new ImmutableCacheStats(6, 7, 8, 9, 10);
+        ImmutableCacheStats expected = new ImmutableCacheStats(7, 9, 11, 13, 15);
+        assertEquals(expected, ImmutableCacheStats.addSnapshots(ics1, ics2));
+    }
+
+    public void testEqualsAndHash() throws Exception {
+        ImmutableCacheStats ics1 = new ImmutableCacheStats(1, 2, 3, 4, 5);
+        ImmutableCacheStats ics2 = new ImmutableCacheStats(1, 2, 3, 4, 5);
+        ImmutableCacheStats ics3 = new ImmutableCacheStats(0, 2, 3, 4, 5);
+
+        assertEquals(ics1, ics2);
+        assertNotEquals(ics1, ics3);
+        assertNotEquals(ics1, null);
+        assertNotEquals(ics1, "string");
+
+        assertEquals(ics1.hashCode(), ics2.hashCode());
+        assertNotEquals(ics1.hashCode(), ics3.hashCode());
+    }
+}
diff --git a/server/src/test/java/org/opensearch/common/cache/store/OpenSearchOnHeapCacheTests.java b/server/src/test/java/org/opensearch/common/cache/store/OpenSearchOnHeapCacheTests.java
new file mode 100644
index 0000000000000..f227db6fee2d1
--- /dev/null
+++ b/server/src/test/java/org/opensearch/common/cache/store/OpenSearchOnHeapCacheTests.java
@@ -0,0 +1,220 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.cache.store;
+
+import org.opensearch.common.Randomness;
+import org.opensearch.common.cache.CacheType;
+import org.opensearch.common.cache.ICache;
+import org.opensearch.common.cache.ICacheKey;
+import org.opensearch.common.cache.LoadAwareCacheLoader;
+import org.opensearch.common.cache.RemovalListener;
+import org.opensearch.common.cache.RemovalNotification;
+import org.opensearch.common.cache.stats.ImmutableCacheStats;
+import org.opensearch.common.cache.stats.ImmutableCacheStatsHolder;
+import org.opensearch.common.cache.store.config.CacheConfig;
+import org.opensearch.common.cache.store.settings.OpenSearchOnHeapCacheSettings;
+import org.opensearch.common.metrics.CounterMetric;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.FeatureFlags;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+import java.util.UUID;
+
+import static org.opensearch.common.cache.store.settings.OpenSearchOnHeapCacheSettings.MAXIMUM_SIZE_IN_BYTES_KEY;
+
+public class OpenSearchOnHeapCacheTests extends OpenSearchTestCase {
+    private final static long keyValueSize = 50;
+    private final static List<String> dimensionNames = List.of("dim1", "dim2", "dim3");
+
+    public void testStats() throws Exception {
+        MockRemovalListener<String, String> listener = new MockRemovalListener<>();
+        int maxKeys = between(10, 50);
+        int numEvicted = between(10, 20);
+        OpenSearchOnHeapCache<String, String> cache = getCache(maxKeys, listener, true, true);
+
+        // When the pluggable caches setting is on, we should get stats as expected from cache.stats().
+
+        List<ICacheKey<String>> keysAdded = new ArrayList<>();
+        int numAdded = maxKeys + numEvicted;
+        for (int i = 0; i < numAdded; i++) {
+            ICacheKey<String> key = getICacheKey(UUID.randomUUID().toString());
+            keysAdded.add(key);
+            cache.computeIfAbsent(key, getLoadAwareCacheLoader());
+
+            assertEquals(i + 1, cache.stats().getTotalMisses());
+            assertEquals(0, cache.stats().getTotalHits());
+            assertEquals(Math.min(maxKeys, i + 1), cache.stats().getTotalItems());
+            assertEquals(Math.min(maxKeys, i + 1) * keyValueSize, cache.stats().getTotalSizeInBytes());
+            assertEquals(Math.max(0, i + 1 - maxKeys), cache.stats().getTotalEvictions());
+        }
+        // do gets from the last part of the list, which should be hits
+        for (int i = numAdded - maxKeys; i < numAdded; i++) {
+            cache.computeIfAbsent(keysAdded.get(i), getLoadAwareCacheLoader());
+            int numHits = i + 1 - (numAdded - maxKeys);
+
+            assertEquals(numAdded, cache.stats().getTotalMisses());
+            assertEquals(numHits, cache.stats().getTotalHits());
+            assertEquals(maxKeys, cache.stats().getTotalItems());
+            assertEquals(maxKeys * keyValueSize, cache.stats().getTotalSizeInBytes());
+            assertEquals(numEvicted, cache.stats().getTotalEvictions());
+        }
+
+        // invalidate keys
+        for (int i = numAdded - maxKeys; i < numAdded; i++) {
+            cache.invalidate(keysAdded.get(i));
+            int numInvalidated = i + 1 - (numAdded - maxKeys);
+
+            assertEquals(numAdded, cache.stats().getTotalMisses());
+            assertEquals(maxKeys, cache.stats().getTotalHits());
+            assertEquals(maxKeys - numInvalidated, cache.stats().getTotalItems());
+            assertEquals((maxKeys - numInvalidated) * keyValueSize, cache.stats().getTotalSizeInBytes());
+            assertEquals(numEvicted, cache.stats().getTotalEvictions());
+        }
+    }
+
+    public void testStatsWithoutPluggableCaches() throws Exception {
+        // When the pluggable caches setting is off, or when we manually set statsTrackingEnabled = false in the config,
+        // we should get all-zero stats from cache.stats(), but count() should still work.
+        MockRemovalListener<String, String> listener = new MockRemovalListener<>();
+        int maxKeys = between(10, 50);
+        int numEvicted = between(10, 20);
+
+        OpenSearchOnHeapCache<String, String> pluggableCachesOffCache = getCache(maxKeys, listener, false, true);
+        OpenSearchOnHeapCache<String, String> manuallySetNoopStatsCache = getCache(maxKeys, listener, true, false);
+        List<OpenSearchOnHeapCache<String, String>> caches = List.of(pluggableCachesOffCache, manuallySetNoopStatsCache);
+
+        for (OpenSearchOnHeapCache<String, String> cache : caches) {
+            int numAdded = maxKeys + numEvicted;
+            for (int i = 0; i < numAdded; i++) {
+                ICacheKey<String> key = getICacheKey(UUID.randomUUID().toString());
+                cache.computeIfAbsent(key, getLoadAwareCacheLoader());
+
+                assertEquals(Math.min(maxKeys, i + 1), cache.count());
+                ImmutableCacheStatsHolder stats = cache.stats();
+                assertZeroStats(cache.stats());
+            }
+        }
+    }
+
+    private void assertZeroStats(ImmutableCacheStatsHolder stats) {
+        assertEquals(new ImmutableCacheStats(0, 0, 0, 0, 0), stats.getTotalStats());
+    }
+
+    private OpenSearchOnHeapCache<String, String> getCache(
+        int maxSizeKeys,
+        MockRemovalListener<String, String> listener,
+        boolean pluggableCachesSetting,
+        boolean statsTrackingEnabled
+    ) {
+        ICache.Factory onHeapCacheFactory = new OpenSearchOnHeapCache.OpenSearchOnHeapCacheFactory();
+        Settings settings = Settings.builder()
+            .put(
+                OpenSearchOnHeapCacheSettings.getSettingListForCacheType(CacheType.INDICES_REQUEST_CACHE)
+                    .get(MAXIMUM_SIZE_IN_BYTES_KEY)
+                    .getKey(),
+                maxSizeKeys * keyValueSize + "b"
+            )
+            .put(FeatureFlags.PLUGGABLE_CACHE, pluggableCachesSetting)
+            .build();
+
+        CacheConfig<String, String> cacheConfig = new CacheConfig.Builder<String, String>().setKeyType(String.class)
+            .setValueType(String.class)
+            .setWeigher((k, v) -> keyValueSize)
+            .setRemovalListener(listener)
+            .setSettings(settings)
+            .setDimensionNames(dimensionNames)
+            .setMaxSizeInBytes(maxSizeKeys * keyValueSize)
+            .setStatsTrackingEnabled(statsTrackingEnabled)
+            .build();
+        return (OpenSearchOnHeapCache<String, String>) onHeapCacheFactory.create(cacheConfig, CacheType.INDICES_REQUEST_CACHE, null);
+    }
+
+    public void testInvalidateWithDropDimensions() throws Exception {
+        MockRemovalListener<String, String> listener = new MockRemovalListener<>();
+        int maxKeys = 50;
+        OpenSearchOnHeapCache<String, String> cache = getCache(maxKeys, listener, true, true);
+
+        List<ICacheKey<String>> keysAdded = new ArrayList<>();
+
+        for (int i = 0; i < maxKeys - 5; i++) {
+            ICacheKey<String> key = new ICacheKey<>(UUID.randomUUID().toString(), getRandomDimensions());
+            keysAdded.add(key);
+            cache.computeIfAbsent(key, getLoadAwareCacheLoader());
+        }
+
+        ICacheKey<String> keyToDrop = keysAdded.get(0);
+        String[] levels = dimensionNames.toArray(new String[0]);
+        ImmutableCacheStats snapshot = cache.stats(levels).getStatsForDimensionValues(keyToDrop.dimensions);
+        assertNotNull(snapshot);
+
+        keyToDrop.setDropStatsForDimensions(true);
+        cache.invalidate(keyToDrop);
+
+        // Now assert the stats are gone for any key that has this combination of dimensions, but still there otherwise
+        for (ICacheKey<String> keyAdded : keysAdded) {
+            snapshot = cache.stats(levels).getStatsForDimensionValues(keyAdded.dimensions);
+            if (keyAdded.dimensions.equals(keyToDrop.dimensions)) {
+                assertNull(snapshot);
+            } else {
+                assertNotNull(snapshot);
+            }
+        }
+    }
+
+    private List<String> getRandomDimensions() {
+        Random rand = Randomness.get();
+        int bound = 3;
+        List<String> result = new ArrayList<>();
+        for (String dimName : dimensionNames) {
+            result.add(String.valueOf(rand.nextInt(bound)));
+        }
+        return result;
+    }
+
+    private static class MockRemovalListener<K, V> implements RemovalListener<ICacheKey<K>, V> {
+        CounterMetric numRemovals;
+
+        MockRemovalListener() {
+            numRemovals = new CounterMetric();
+        }
+
+        @Override
+        public void onRemoval(RemovalNotification<ICacheKey<K>, V> notification) {
+            numRemovals.inc();
+        }
+    }
+
+    private ICacheKey<String> getICacheKey(String key) {
+        List<String> dims = new ArrayList<>();
+        for (String dimName : dimensionNames) {
+            dims.add("0");
+        }
+        return new ICacheKey<>(key, dims);
+    }
+
+    private LoadAwareCacheLoader<ICacheKey<String>, String> getLoadAwareCacheLoader() {
+        return new LoadAwareCacheLoader<>() {
+            boolean isLoaded = false;
+
+            @Override
+            public String load(ICacheKey<String> key) {
+                isLoaded = true;
+                return UUID.randomUUID().toString();
+            }
+
+            @Override
+            public boolean isLoaded() {
+                return isLoaded;
+            }
+        };
+    }
+}
diff --git a/server/src/test/java/org/opensearch/common/hashing/FNV1aTests.java b/server/src/test/java/org/opensearch/common/hashing/FNV1aTests.java
new file mode 100644
index 0000000000000..8d41211f10134
--- /dev/null
+++ b/server/src/test/java/org/opensearch/common/hashing/FNV1aTests.java
@@ -0,0 +1,48 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.common.hashing;
+
+import org.opensearch.common.hash.FNV1a;
+import org.opensearch.test.OpenSearchTestCase;
+
+public class FNV1aTests extends OpenSearchTestCase {
+
+    public void testHash32WithKnownValues() {
+        assertEquals(-1114940029532279145L, FNV1a.hash32("1sH3kJO5TyeskNekv2YTbA0segmentsdata"));
+        assertEquals(-5313793557685118702L, FNV1a.hash32("1sH3kJO5TyeskNekv2YTbA0segmentsmetadata"));
+        assertEquals(-4776941653547780179L, FNV1a.hash32("1sH3kJO5TyeskNekv2YTbA0translogdata"));
+        assertEquals(7773876801598345624L, FNV1a.hash32("1sH3kJO5TyeskNekv2YTbA0translogmetadata"));
+        assertEquals(3174284101845744576L, FNV1a.hash32("1sH3kJO5TyeskNekv2YTbA0segmentslock_files"));
+        assertEquals(875447599647258598L, FNV1a.hash32("hell"));
+        assertEquals(1460560186469985451L, FNV1a.hash32("hello"));
+        assertEquals(-4959477702557352110L, FNV1a.hash32("hello w"));
+        assertEquals(-777130915070571257L, FNV1a.hash32("hello wo"));
+        assertEquals(-7887204531510399185L, FNV1a.hash32("hello wor"));
+        assertEquals(-782004333700192647L, FNV1a.hash32("hello worl"));
+        assertEquals(2168278929747165095L, FNV1a.hash32("hello world"));
+        assertEquals(2655121221658607504L, FNV1a.hash32("The quick brown fox jumps over the lazy dog"));
+    }
+
+    public void testHash64WithKnownValues() {
+        assertEquals(-8975854101357662761L, FNV1a.hash64("1sH3kJO5TyeskNekv2YTbA0segmentsdata"));
+        assertEquals(-4380291990281602606L, FNV1a.hash64("1sH3kJO5TyeskNekv2YTbA0segmentsmetadata"));
+        assertEquals(-4532418109365814419L, FNV1a.hash64("1sH3kJO5TyeskNekv2YTbA0translogdata"));
+        assertEquals(41331743556869080L, FNV1a.hash64("1sH3kJO5TyeskNekv2YTbA0translogmetadata"));
+        assertEquals(6170437157231275808L, FNV1a.hash64("1sH3kJO5TyeskNekv2YTbA0segmentslock_files"));
+        assertEquals(763638091547294502L, FNV1a.hash64("hell"));
+        assertEquals(-6615550055289275125L, FNV1a.hash64("hello"));
+        assertEquals(-8428874042178798254L, FNV1a.hash64("hello w"));
+        assertEquals(-6323438951910650201L, FNV1a.hash64("hello wo"));
+        assertEquals(7042426588567368687L, FNV1a.hash64("hello wor"));
+        assertEquals(7273314957493782425L, FNV1a.hash64("hello worl"));
+        assertEquals(8618312879776256743L, FNV1a.hash64("hello world"));
+        assertEquals(-866459186506731248L, FNV1a.hash64("The quick brown fox jumps over the lazy dog"));
+    }
+
+}
diff --git a/server/src/test/java/org/opensearch/common/network/NetworkModuleTests.java b/server/src/test/java/org/opensearch/common/network/NetworkModuleTests.java
index 1c607ca0dc98b..447377e372e61 100644
--- a/server/src/test/java/org/opensearch/common/network/NetworkModuleTests.java
+++ b/server/src/test/java/org/opensearch/common/network/NetworkModuleTests.java
@@ -47,13 +47,15 @@
 import org.opensearch.http.HttpStats;
 import org.opensearch.http.NullDispatcher;
 import org.opensearch.plugins.NetworkPlugin;
+import org.opensearch.plugins.SecureHttpTransportSettingsProvider;
+import org.opensearch.plugins.SecureSettingsFactory;
 import org.opensearch.plugins.SecureTransportSettingsProvider;
+import org.opensearch.plugins.TransportExceptionHandler;
 import org.opensearch.telemetry.tracing.Tracer;
 import org.opensearch.telemetry.tracing.noop.NoopTracer;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.threadpool.TestThreadPool;
 import org.opensearch.threadpool.ThreadPool;
-import org.opensearch.transport.TcpTransport;
 import org.opensearch.transport.Transport;
 import org.opensearch.transport.TransportInterceptor;
 import org.opensearch.transport.TransportRequest;
@@ -73,38 +75,60 @@
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Supplier;
 
+import static org.hamcrest.CoreMatchers.startsWith;
+
 public class NetworkModuleTests extends OpenSearchTestCase {
     private ThreadPool threadPool;
-    private SecureTransportSettingsProvider secureTransportSettingsProvider;
+    private SecureSettingsFactory secureSettingsFactory;
 
     @Override
     public void setUp() throws Exception {
         super.setUp();
         threadPool = new TestThreadPool(NetworkModuleTests.class.getName());
-        secureTransportSettingsProvider = new SecureTransportSettingsProvider() {
-            @Override
-            public Optional<ServerExceptionHandler> buildHttpServerExceptionHandler(Settings settings, HttpServerTransport transport) {
-                return Optional.empty();
-            }
-
-            @Override
-            public Optional<ServerExceptionHandler> buildServerTransportExceptionHandler(Settings settings, TcpTransport transport) {
-                return Optional.empty();
-            }
-
-            @Override
-            public Optional<SSLEngine> buildSecureHttpServerEngine(Settings settings, HttpServerTransport transport) throws SSLException {
-                return Optional.empty();
-            }
+        secureSettingsFactory = new SecureSettingsFactory() {
 
             @Override
-            public Optional<SSLEngine> buildSecureServerTransportEngine(Settings settings, TcpTransport transport) throws SSLException {
-                return Optional.empty();
+            public Optional<SecureTransportSettingsProvider> getSecureTransportSettingsProvider(Settings settings) {
+                return Optional.of(new SecureTransportSettingsProvider() {
+                    @Override
+                    public Optional<TransportExceptionHandler> buildServerTransportExceptionHandler(
+                        Settings settings,
+                        Transport transport
+                    ) {
+                        return Optional.empty();
+                    }
+
+                    @Override
+                    public Optional<SSLEngine> buildSecureServerTransportEngine(Settings settings, Transport transport)
+                        throws SSLException {
+                        return Optional.empty();
+                    }
+
+                    @Override
+                    public Optional<SSLEngine> buildSecureClientTransportEngine(Settings settings, String hostname, int port)
+                        throws SSLException {
+                        return Optional.empty();
+                    }
+                });
             }
 
             @Override
-            public Optional<SSLEngine> buildSecureClientTransportEngine(Settings settings, String hostname, int port) throws SSLException {
-                return Optional.empty();
+            public Optional<SecureHttpTransportSettingsProvider> getSecureHttpTransportSettingsProvider(Settings settings) {
+                return Optional.of(new SecureHttpTransportSettingsProvider() {
+                    @Override
+                    public Optional<SSLEngine> buildSecureHttpServerEngine(Settings settings, HttpServerTransport transport)
+                        throws SSLException {
+                        return Optional.empty();
+                    }
+
+                    @Override
+                    public Optional<TransportExceptionHandler> buildHttpServerExceptionHandler(
+                        Settings settings,
+                        HttpServerTransport transport
+                    ) {
+                        return Optional.empty();
+                    }
+                });
             }
         };
     }
@@ -211,7 +235,7 @@ public Map<String, Supplier<Transport>> getSecureTransports(
                 return Collections.singletonMap("custom-secure", custom);
             }
         };
-        NetworkModule module = newNetworkModule(settings, null, List.of(secureTransportSettingsProvider), plugin);
+        NetworkModule module = newNetworkModule(settings, null, List.of(secureSettingsFactory), plugin);
         assertSame(custom, module.getTransportSupplier());
     }
 
@@ -222,7 +246,7 @@ public void testRegisterSecureHttpTransport() {
             .build();
         Supplier<HttpServerTransport> custom = FakeHttpTransport::new;
 
-        NetworkModule module = newNetworkModule(settings, null, List.of(secureTransportSettingsProvider), new NetworkPlugin() {
+        NetworkModule module = newNetworkModule(settings, null, List.of(secureSettingsFactory), new NetworkPlugin() {
             @Override
             public Map<String, Supplier<HttpServerTransport>> getSecureHttpTransports(
                 Settings settings,
@@ -234,7 +258,7 @@ public Map<String, Supplier<HttpServerTransport>> getSecureHttpTransports(
                 NetworkService networkService,
                 HttpServerTransport.Dispatcher requestDispatcher,
                 ClusterSettings clusterSettings,
-                SecureTransportSettingsProvider secureTransportSettingsProvider,
+                SecureHttpTransportSettingsProvider secureTransportSettingsProvider,
                 Tracer tracer
             ) {
                 return Collections.singletonMap("custom-secure", custom);
@@ -595,7 +619,7 @@ private NetworkModule newNetworkModule(
     private NetworkModule newNetworkModule(
         Settings settings,
         List<TransportInterceptor> coreTransportInterceptors,
-        List<SecureTransportSettingsProvider> secureTransportSettingsProviders,
+        List<SecureSettingsFactory> secureSettingsFactories,
         NetworkPlugin... plugins
     ) {
         return new NetworkModule(
@@ -612,7 +636,33 @@ private NetworkModule newNetworkModule(
             new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
             NoopTracer.INSTANCE,
             coreTransportInterceptors,
-            secureTransportSettingsProviders
+            secureSettingsFactories
+        );
+    }
+
+    public void testRegisterSecureTransportMultipleProviers() {
+        Settings settings = Settings.builder().put(NetworkModule.TRANSPORT_TYPE_KEY, "custom-secure").build();
+        Supplier<Transport> custom = () -> null; // content doesn't matter we check reference equality
+        NetworkPlugin plugin = new NetworkPlugin() {
+            @Override
+            public Map<String, Supplier<Transport>> getSecureTransports(
+                Settings settings,
+                ThreadPool threadPool,
+                PageCacheRecycler pageCacheRecycler,
+                CircuitBreakerService circuitBreakerService,
+                NamedWriteableRegistry namedWriteableRegistry,
+                NetworkService networkService,
+                SecureTransportSettingsProvider secureTransportSettingsProvider,
+                Tracer tracer
+            ) {
+                return Collections.singletonMap("custom-secure", custom);
+            }
+        };
+
+        final IllegalArgumentException ex = assertThrows(
+            IllegalArgumentException.class,
+            () -> newNetworkModule(settings, null, List.of(secureSettingsFactory, secureSettingsFactory), plugin)
         );
+        assertThat(ex.getMessage(), startsWith("there is more than one secure transport settings provider"));
     }
 }
diff --git a/server/src/test/java/org/opensearch/common/settings/SettingTests.java b/server/src/test/java/org/opensearch/common/settings/SettingTests.java
index c6da96b521276..7ebee680e8e52 100644
--- a/server/src/test/java/org/opensearch/common/settings/SettingTests.java
+++ b/server/src/test/java/org/opensearch/common/settings/SettingTests.java
@@ -201,7 +201,7 @@ public void testMemorySize() {
         assertEquals(new ByteSizeValue(12), value.get());
 
         assertTrue(settingUpdater.apply(Settings.builder().put("a.byte.size", "20%").build(), Settings.EMPTY));
-        assertEquals(new ByteSizeValue((int) (JvmInfo.jvmInfo().getMem().getHeapMax().getBytes() * 0.2)), value.get());
+        assertEquals(new ByteSizeValue((long) (JvmInfo.jvmInfo().getMem().getHeapMax().getBytes() * 0.2)), value.get());
     }
 
     public void testMemorySizeWithFallbackValue() {
@@ -219,10 +219,12 @@ public void testMemorySizeWithFallbackValue() {
         assertEquals(memorySizeValue.getBytes(), JvmInfo.jvmInfo().getMem().getHeapMax().getBytes() * 0.2, 1.0);
 
         assertTrue(settingUpdater.apply(Settings.builder().put("a.byte.size", "30%").build(), Settings.EMPTY));
-        assertEquals(new ByteSizeValue((int) (JvmInfo.jvmInfo().getMem().getHeapMax().getBytes() * 0.3)), value.get());
+        // If value=getHeapMax()*0.3 is bigger than 2gb, and is bigger than Integer.MAX_VALUE,
+        // then (long)((int) value) will lose precision.
+        assertEquals(new ByteSizeValue((long) (JvmInfo.jvmInfo().getMem().getHeapMax().getBytes() * 0.3)), value.get());
 
         assertTrue(settingUpdater.apply(Settings.builder().put("b.byte.size", "40%").build(), Settings.EMPTY));
-        assertEquals(new ByteSizeValue((int) (JvmInfo.jvmInfo().getMem().getHeapMax().getBytes() * 0.4)), value.get());
+        assertEquals(new ByteSizeValue((long) (JvmInfo.jvmInfo().getMem().getHeapMax().getBytes() * 0.4)), value.get());
     }
 
     public void testSimpleUpdate() {
diff --git a/server/src/test/java/org/opensearch/common/time/DateUtilsTests.java b/server/src/test/java/org/opensearch/common/time/DateUtilsTests.java
index d9662d1de9e0c..98a79f3ca38dc 100644
--- a/server/src/test/java/org/opensearch/common/time/DateUtilsTests.java
+++ b/server/src/test/java/org/opensearch/common/time/DateUtilsTests.java
@@ -58,7 +58,7 @@
 import static org.hamcrest.Matchers.is;
 
 public class DateUtilsTests extends OpenSearchTestCase {
-    private static final Set<String> IGNORE = new HashSet<>(Arrays.asList("America/Ciudad_Juarez"));
+    private static final Set<String> IGNORE = new HashSet<>(Arrays.asList("Antarctica/Vostok"));
 
     public void testTimezoneIds() {
         assertNull(DateUtils.dateTimeZoneToZoneId(null));
diff --git a/server/src/test/java/org/opensearch/common/util/FeatureFlagTests.java b/server/src/test/java/org/opensearch/common/util/FeatureFlagTests.java
index f175308482b15..6b6c7b96266d3 100644
--- a/server/src/test/java/org/opensearch/common/util/FeatureFlagTests.java
+++ b/server/src/test/java/org/opensearch/common/util/FeatureFlagTests.java
@@ -8,9 +8,14 @@
 
 package org.opensearch.common.util;
 
+import org.opensearch.common.settings.Settings;
 import org.opensearch.test.FeatureFlagSetter;
 import org.opensearch.test.OpenSearchTestCase;
 
+import static org.opensearch.common.util.FeatureFlags.DATETIME_FORMATTER_CACHING;
+import static org.opensearch.common.util.FeatureFlags.EXTENSIONS;
+import static org.opensearch.common.util.FeatureFlags.IDENTITY;
+
 public class FeatureFlagTests extends OpenSearchTestCase {
 
     private final String FLAG_PREFIX = "opensearch.experimental.feature.";
@@ -33,4 +38,27 @@ public void testNonBooleanFeatureFlag() {
         assertNotNull(System.getProperty(javaVersionProperty));
         assertFalse(FeatureFlags.isEnabled(javaVersionProperty));
     }
+
+    public void testBooleanFeatureFlagWithDefaultSetToFalse() {
+        final String testFlag = IDENTITY;
+        FeatureFlags.initializeFeatureFlags(Settings.EMPTY);
+        assertNotNull(testFlag);
+        assertFalse(FeatureFlags.isEnabled(testFlag));
+    }
+
+    public void testBooleanFeatureFlagInitializedWithEmptySettingsAndDefaultSetToFalse() {
+        final String testFlag = DATETIME_FORMATTER_CACHING;
+        FeatureFlags.initializeFeatureFlags(Settings.EMPTY);
+        assertNotNull(testFlag);
+        assertFalse(FeatureFlags.isEnabled(testFlag));
+    }
+
+    public void testInitializeFeatureFlagsWithExperimentalSettings() {
+        FeatureFlags.initializeFeatureFlags(Settings.builder().put(IDENTITY, true).build());
+        assertTrue(FeatureFlags.isEnabled(IDENTITY));
+        assertFalse(FeatureFlags.isEnabled(DATETIME_FORMATTER_CACHING));
+        assertFalse(FeatureFlags.isEnabled(EXTENSIONS));
+        // reset FeatureFlags to defaults
+        FeatureFlags.initializeFeatureFlags(Settings.EMPTY);
+    }
 }
diff --git a/server/src/test/java/org/opensearch/gateway/AsyncShardFetchTests.java b/server/src/test/java/org/opensearch/gateway/AsyncShardFetchTests.java
index 4e5e9c71e1fe4..db97c3ece94ba 100644
--- a/server/src/test/java/org/opensearch/gateway/AsyncShardFetchTests.java
+++ b/server/src/test/java/org/opensearch/gateway/AsyncShardFetchTests.java
@@ -39,7 +39,6 @@
 import org.opensearch.cluster.node.DiscoveryNodeRole;
 import org.opensearch.cluster.node.DiscoveryNodes;
 import org.opensearch.core.index.shard.ShardId;
-import org.opensearch.indices.store.ShardAttributes;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.threadpool.TestThreadPool;
 import org.opensearch.threadpool.ThreadPool;
@@ -47,7 +46,6 @@
 import org.junit.Before;
 
 import java.util.Collections;
-import java.util.HashMap;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.CountDownLatch;
@@ -86,16 +84,7 @@ public class AsyncShardFetchTests extends OpenSearchTestCase {
     public void setUp() throws Exception {
         super.setUp();
         this.threadPool = new TestThreadPool(getTestName());
-        if (randomBoolean()) {
-            this.test = new TestFetch(threadPool);
-        } else {
-            HashMap<ShardId, ShardAttributes> shardToCustomDataPath = new HashMap<>();
-            ShardId shardId0 = new ShardId("index1", "index_uuid1", 0);
-            ShardId shardId1 = new ShardId("index2", "index_uuid2", 0);
-            shardToCustomDataPath.put(shardId0, new ShardAttributes(shardId0, ""));
-            shardToCustomDataPath.put(shardId1, new ShardAttributes(shardId1, ""));
-            this.test = new TestFetch(threadPool, shardToCustomDataPath);
-        }
+        this.test = new TestFetch(threadPool);
     }
 
     @After
@@ -414,11 +403,6 @@ static class Entry {
             this.threadPool = threadPool;
         }
 
-        TestFetch(ThreadPool threadPool, Map<ShardId, ShardAttributes> shardAttributesMap) {
-            super(LogManager.getLogger(TestFetch.class), "test", shardAttributesMap, null, "test-batch");
-            this.threadPool = threadPool;
-        }
-
         public void addSimulation(String nodeId, Response response) {
             simulations.put(nodeId, new Entry(response, null));
         }
diff --git a/server/src/test/java/org/opensearch/gateway/BatchTestUtil.java b/server/src/test/java/org/opensearch/gateway/BatchTestUtil.java
new file mode 100644
index 0000000000000..69f0cfeeb2c7d
--- /dev/null
+++ b/server/src/test/java/org/opensearch/gateway/BatchTestUtil.java
@@ -0,0 +1,25 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.gateway;
+
+import org.opensearch.core.index.shard.ShardId;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class BatchTestUtil {
+    public static List<ShardId> setUpShards(int numberOfShards) {
+        List<ShardId> shards = new ArrayList<>();
+        for (int shardNumber = 0; shardNumber < numberOfShards; shardNumber++) {
+            ShardId shardId = new ShardId("test", "_na_", shardNumber);
+            shards.add(shardId);
+        }
+        return shards;
+    }
+}
diff --git a/server/src/test/java/org/opensearch/gateway/GatewayAllocatorTests.java b/server/src/test/java/org/opensearch/gateway/GatewayAllocatorTests.java
new file mode 100644
index 0000000000000..bb59a5792ec8c
--- /dev/null
+++ b/server/src/test/java/org/opensearch/gateway/GatewayAllocatorTests.java
@@ -0,0 +1,360 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.gateway;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.opensearch.Version;
+import org.opensearch.action.support.nodes.BaseNodeResponse;
+import org.opensearch.cluster.ClusterInfo;
+import org.opensearch.cluster.ClusterName;
+import org.opensearch.cluster.ClusterState;
+import org.opensearch.cluster.OpenSearchAllocationTestCase;
+import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.cluster.metadata.Metadata;
+import org.opensearch.cluster.routing.IndexShardRoutingTable;
+import org.opensearch.cluster.routing.RoutingNodes;
+import org.opensearch.cluster.routing.RoutingTable;
+import org.opensearch.cluster.routing.ShardRouting;
+import org.opensearch.cluster.routing.allocation.RoutingAllocation;
+import org.opensearch.cluster.routing.allocation.decider.AllocationDeciders;
+import org.opensearch.common.collect.Tuple;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.core.index.shard.ShardId;
+import org.opensearch.snapshots.SnapshotShardSizeInfo;
+import org.opensearch.test.gateway.TestShardBatchGatewayAllocator;
+import org.junit.Before;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public class GatewayAllocatorTests extends OpenSearchAllocationTestCase {
+
+    private final Logger logger = LogManager.getLogger(GatewayAllocatorTests.class);
+    TestShardBatchGatewayAllocator testShardsBatchGatewayAllocator = null;
+    ClusterState clusterState = null;
+    RoutingAllocation testAllocation = null;
+    String indexPrefix = "TEST";
+
+    @Override
+    @Before
+    public void setUp() throws Exception {
+        super.setUp();
+        testShardsBatchGatewayAllocator = new TestShardBatchGatewayAllocator();
+    }
+
+    public void testSingleBatchCreation() {
+        createIndexAndUpdateClusterState(1, 3, 1);
+        createBatchesAndAssert(1);
+    }
+
+    public void testTwoBatchCreation() {
+        createIndexAndUpdateClusterState(2, 1020, 1);
+        createBatchesAndAssert(2);
+
+        List<ShardsBatchGatewayAllocator.ShardsBatch> listOfBatches = new ArrayList<>(
+            testShardsBatchGatewayAllocator.getBatchIdToStartedShardBatch().values()
+        );
+        assertNotEquals(listOfBatches.get(0), listOfBatches.get(1));
+
+        // test for replicas
+        listOfBatches = new ArrayList<>(testShardsBatchGatewayAllocator.getBatchIdToStoreShardBatch().values());
+        assertNotEquals(listOfBatches.get(0), listOfBatches.get(1));
+    }
+
+    public void testNonDuplicationOfBatch() {
+        createIndexAndUpdateClusterState(1, 3, 1);
+        Tuple<Set<String>, Set<String>> batches = createBatchesAndAssert(1);
+        assertEquals(1, batches.v1().size());
+        assertEquals(1, batches.v2().size());
+
+        // again try to create batch and verify no new batch is created since shard is already batched and no new unassigned shard
+        assertEquals(batches.v1(), testShardsBatchGatewayAllocator.createAndUpdateBatches(testAllocation, true));
+        assertEquals(batches.v2(), testShardsBatchGatewayAllocator.createAndUpdateBatches(testAllocation, false));
+    }
+
+    public void testCorrectnessOfBatch() {
+        createIndexAndUpdateClusterState(2, 1020, 1);
+        createBatchesAndAssert(2);
+        Set<ShardId> shardsSet1 = clusterState.routingTable()
+            .index(indexPrefix + 0)
+            .getShards()
+            .values()
+            .stream()
+            .map(IndexShardRoutingTable::getShardId)
+            .collect(Collectors.toSet());
+        Set<ShardId> shardsSet2 = clusterState.routingTable()
+            .index(indexPrefix + 1)
+            .getShards()
+            .values()
+            .stream()
+            .map(IndexShardRoutingTable::getShardId)
+            .collect(Collectors.toSet());
+        shardsSet1.addAll(shardsSet2);
+
+        Set<ShardId> shardsInAllbatches = testShardsBatchGatewayAllocator.getBatchIdToStartedShardBatch()
+            .values()
+            .stream()
+            .map(ShardsBatchGatewayAllocator.ShardsBatch::getBatchedShards)
+            .flatMap(Set::stream)
+            .collect(Collectors.toSet());
+        assertEquals(shardsInAllbatches, shardsSet1);
+        shardsInAllbatches = testShardsBatchGatewayAllocator.getBatchIdToStoreShardBatch()
+            .values()
+            .stream()
+            .map(ShardsBatchGatewayAllocator.ShardsBatch::getBatchedShards)
+            .flatMap(Set::stream)
+            .collect(Collectors.toSet());
+        assertEquals(shardsInAllbatches, shardsSet1);
+
+        Set<ShardRouting> primariesInAllBatches = testShardsBatchGatewayAllocator.getBatchIdToStartedShardBatch()
+            .values()
+            .stream()
+            .map(ShardsBatchGatewayAllocator.ShardsBatch::getBatchedShardRoutings)
+            .flatMap(List::stream)
+            .collect(Collectors.toSet());
+        primariesInAllBatches.forEach(shardRouting -> assertTrue(shardRouting.unassigned() && shardRouting.primary() == true));
+
+        Set<ShardRouting> replicasInAllBatches = testShardsBatchGatewayAllocator.getBatchIdToStoreShardBatch()
+            .values()
+            .stream()
+            .map(ShardsBatchGatewayAllocator.ShardsBatch::getBatchedShardRoutings)
+            .flatMap(List::stream)
+            .collect(Collectors.toSet());
+
+        replicasInAllBatches.forEach(shardRouting -> assertTrue(shardRouting.unassigned() && shardRouting.primary() == false));
+    }
+
+    public void testAsyncFetcherCreationInBatch() {
+        createIndexAndUpdateClusterState(1, 3, 1);
+        Tuple<Set<String>, Set<String>> batchesTuple = createBatchesAndAssert(1);
+        Set<String> primaryBatches = batchesTuple.v1();
+        Set<String> replicaBatches = batchesTuple.v2();
+
+        ShardsBatchGatewayAllocator.ShardsBatch shardsBatch = testShardsBatchGatewayAllocator.getBatchIdToStartedShardBatch()
+            .get(primaryBatches.iterator().next());
+        AsyncShardFetch<? extends BaseNodeResponse> asyncFetcher = shardsBatch.getAsyncFetcher();
+        // assert asyncFetcher is not null
+        assertNotNull(asyncFetcher);
+        shardsBatch = testShardsBatchGatewayAllocator.getBatchIdToStoreShardBatch().get(replicaBatches.iterator().next());
+        asyncFetcher = shardsBatch.getAsyncFetcher();
+        assertNotNull(asyncFetcher);
+    }
+
+    public void testSafelyRemoveShardFromBatch() {
+        createIndexAndUpdateClusterState(2, 1023, 1);
+
+        Tuple<Set<String>, Set<String>> batchesTuple = createBatchesAndAssert(2);
+        Set<String> primaryBatches = batchesTuple.v1();
+        Set<String> replicaBatches = batchesTuple.v2();
+
+        ShardsBatchGatewayAllocator.ShardsBatch primaryShardsBatch = testShardsBatchGatewayAllocator.getBatchIdToStartedShardBatch()
+            .get(primaryBatches.iterator().next());
+        ShardRouting primaryShardRouting = primaryShardsBatch.getBatchedShardRoutings().iterator().next();
+        assertEquals(2, replicaBatches.size());
+        ShardsBatchGatewayAllocator.ShardsBatch replicaShardsBatch = testShardsBatchGatewayAllocator.getBatchIdToStoreShardBatch()
+            .get(replicaBatches.iterator().next());
+        ShardRouting replicaShardRouting = replicaShardsBatch.getBatchedShardRoutings().iterator().next();
+
+        // delete 1 shard routing from each batch
+        testShardsBatchGatewayAllocator.safelyRemoveShardFromBatch(primaryShardRouting);
+
+        testShardsBatchGatewayAllocator.safelyRemoveShardFromBatch(replicaShardRouting);
+        // verify that shard routing is removed from both batches
+        assertFalse(primaryShardsBatch.getBatchedShards().contains(primaryShardRouting.shardId()));
+        assertFalse(replicaShardsBatch.getBatchedShards().contains(replicaShardRouting.shardId()));
+
+        // try to remove that shard again to see if its no op and doent result in exception
+        testShardsBatchGatewayAllocator.safelyRemoveShardFromBatch(primaryShardRouting);
+        testShardsBatchGatewayAllocator.safelyRemoveShardFromBatch(replicaShardRouting);
+
+        // now remove all shard routings to verify that batch only gets deleted
+        primaryShardsBatch.getBatchedShardRoutings().forEach(testShardsBatchGatewayAllocator::safelyRemoveShardFromBatch);
+        replicaShardsBatch.getBatchedShardRoutings().forEach(testShardsBatchGatewayAllocator::safelyRemoveShardFromBatch);
+
+        assertFalse(testShardsBatchGatewayAllocator.getBatchIdToStartedShardBatch().containsKey(primaryShardsBatch.getBatchId()));
+        assertFalse(testShardsBatchGatewayAllocator.getBatchIdToStoreShardBatch().containsKey(replicaShardsBatch.getBatchId()));
+        assertEquals(1, testShardsBatchGatewayAllocator.getBatchIdToStartedShardBatch().size());
+        assertEquals(1, testShardsBatchGatewayAllocator.getBatchIdToStoreShardBatch().size());
+    }
+
+    public void testSafelyRemoveShardFromBothBatch() {
+        createIndexAndUpdateClusterState(1, 3, 1);
+        createBatchesAndAssert(1);
+        ShardsBatchGatewayAllocator.ShardsBatch primaryShardsBatch = testShardsBatchGatewayAllocator.getBatchIdToStartedShardBatch()
+            .values()
+            .iterator()
+            .next();
+        ShardsBatchGatewayAllocator.ShardsBatch replicaShardsBatch = testShardsBatchGatewayAllocator.getBatchIdToStoreShardBatch()
+            .values()
+            .iterator()
+            .next();
+
+        ShardRouting anyPrimary = primaryShardsBatch.getBatchedShardRoutings().iterator().next();
+        // remove first shard routing from both batches
+        testShardsBatchGatewayAllocator.safelyRemoveShardFromBothBatch(anyPrimary);
+
+        // verify that shard routing is removed from both batches
+        assertFalse(primaryShardsBatch.getBatchedShards().contains(anyPrimary.shardId()));
+        assertFalse(replicaShardsBatch.getBatchedShards().contains(anyPrimary.shardId()));
+
+        // try to remove that shard again to see if its no op and doesnt result in exception
+        testShardsBatchGatewayAllocator.safelyRemoveShardFromBothBatch(anyPrimary);
+
+        // now remove all shard routings to verify that batch gets deleted
+        primaryShardsBatch.getBatchedShardRoutings().forEach(testShardsBatchGatewayAllocator::safelyRemoveShardFromBothBatch);
+        replicaShardsBatch.getBatchedShardRoutings().forEach(testShardsBatchGatewayAllocator::safelyRemoveShardFromBothBatch);
+
+        assertFalse(testShardsBatchGatewayAllocator.getBatchIdToStartedShardBatch().containsKey(primaryShardsBatch.getBatchId()));
+        assertFalse(testShardsBatchGatewayAllocator.getBatchIdToStoreShardBatch().containsKey(replicaShardsBatch.getBatchId()));
+        assertEquals(0, testShardsBatchGatewayAllocator.getBatchIdToStartedShardBatch().size());
+        assertEquals(0, testShardsBatchGatewayAllocator.getBatchIdToStoreShardBatch().size());
+    }
+
+    public void testGetBatchIdExisting() {
+        createIndexAndUpdateClusterState(2, 1020, 1);
+        // get all shardsRoutings for test index
+        List<ShardRouting> allShardRoutings1 = clusterState.routingTable()
+            .index(indexPrefix + 0)
+            .getShards()
+            .values()
+            .stream()
+            .map(IndexShardRoutingTable::getShards)
+            .flatMap(List::stream)
+            .collect(Collectors.toList());
+        List<ShardRouting> allShardRouting2 = clusterState.routingTable()
+            .index(indexPrefix + 1)
+            .getShards()
+            .values()
+            .stream()
+            .map(IndexShardRoutingTable::getShards)
+            .flatMap(List::stream)
+            .collect(Collectors.toList());
+
+        Tuple<Set<String>, Set<String>> batchesTuple = createBatchesAndAssert(2);
+        Set<String> primaryBatches = batchesTuple.v1();
+        Set<String> replicaBatches = batchesTuple.v2();
+
+        // create a map of shards to batch id for primaries
+
+        Map<ShardId, String> shardIdToBatchIdForStartedShards = new HashMap<>();
+        allShardRoutings1.addAll(allShardRouting2);
+        assertEquals(4080, allShardRoutings1.size());
+        for (ShardRouting shardRouting : allShardRoutings1) {
+            for (String batchId : primaryBatches) {
+                if (shardRouting.primary() == true
+                    && testShardsBatchGatewayAllocator.getBatchIdToStartedShardBatch()
+                        .get(batchId)
+                        .getBatchedShards()
+                        .contains(shardRouting.shardId())) {
+                    if (shardIdToBatchIdForStartedShards.containsKey(shardRouting.shardId())) {
+                        fail("found duplicate shard routing for shard. One shard cant be in multiple batches " + shardRouting.shardId());
+                    }
+                    assertTrue(shardRouting.primary());
+                    shardIdToBatchIdForStartedShards.put(shardRouting.shardId(), batchId);
+                }
+            }
+        }
+        Map<ShardId, String> shardIdToBatchIdForStoreShards = new HashMap<>();
+
+        for (ShardRouting shardRouting : allShardRoutings1) {
+            for (String batchId : replicaBatches) {
+                if (shardRouting.primary() == false
+                    && testShardsBatchGatewayAllocator.getBatchIdToStoreShardBatch()
+                        .get(batchId)
+                        .getBatchedShards()
+                        .contains(shardRouting.shardId())) {
+                    if (shardIdToBatchIdForStoreShards.containsKey(shardRouting.shardId())) {
+                        fail("found duplicate shard routing for shard. One shard cant be in multiple batches " + shardRouting.shardId());
+                    }
+                    assertFalse(shardRouting.primary());
+                    shardIdToBatchIdForStoreShards.put(shardRouting.shardId(), batchId);
+                }
+            }
+        }
+
+        assertEquals(4080, shardIdToBatchIdForStartedShards.size() + shardIdToBatchIdForStoreShards.size());
+        // now compare the maps with getBatchId() call
+        for (ShardRouting shardRouting : allShardRoutings1) {
+            if (shardRouting.primary()) {
+                assertEquals(
+                    shardIdToBatchIdForStartedShards.get(shardRouting.shardId()),
+                    testShardsBatchGatewayAllocator.getBatchId(shardRouting, true)
+                );
+            } else {
+                assertEquals(
+                    shardIdToBatchIdForStoreShards.get(shardRouting.shardId()),
+                    testShardsBatchGatewayAllocator.getBatchId(shardRouting, false)
+                );
+            }
+        }
+    }
+
+    public void testGetBatchIdNonExisting() {
+        createIndexAndUpdateClusterState(1, 1, 1);
+        List<ShardRouting> allShardRoutings = clusterState.routingTable()
+            .index(indexPrefix + 0)
+            .getShards()
+            .values()
+            .stream()
+            .map(IndexShardRoutingTable::getShards)
+            .flatMap(List::stream)
+            .collect(Collectors.toList());
+        allShardRoutings.forEach(shard -> assertNull(testShardsBatchGatewayAllocator.getBatchId(shard, shard.primary())));
+    }
+
+    private void createIndexAndUpdateClusterState(int count, int numberOfShards, int numberOfReplicas) {
+        if (count == 0) return;
+        Metadata.Builder metadata = Metadata.builder();
+        RoutingTable.Builder routingTableBuilder = RoutingTable.builder();
+        for (int i = 0; i < count; i++) {
+            String indexName = indexPrefix + i;
+            metadata.put(
+                IndexMetadata.builder(indexName)
+                    .settings(settings(Version.CURRENT))
+                    .numberOfShards(numberOfShards)
+                    .numberOfReplicas(numberOfReplicas)
+            );
+        }
+        for (int i = 0; i < count; i++) {
+            String indexName = indexPrefix + i;
+            routingTableBuilder = routingTableBuilder.addAsNew(metadata.build().index(indexName));
+        }
+        clusterState = ClusterState.builder(ClusterName.CLUSTER_NAME_SETTING.getDefault(Settings.EMPTY))
+            .metadata(metadata.build())
+            .routingTable(routingTableBuilder.build())
+            .build();
+        testAllocation = new RoutingAllocation(
+            new AllocationDeciders(Collections.emptyList()),
+            new RoutingNodes(clusterState, false),
+            clusterState,
+            ClusterInfo.EMPTY,
+            SnapshotShardSizeInfo.EMPTY,
+            System.nanoTime()
+        );
+    }
+
+    // call this after index creation and update cluster state
+    private Tuple<Set<String>, Set<String>> createBatchesAndAssert(int expectedBatchSize) {
+        Set<String> primaryBatches = testShardsBatchGatewayAllocator.createAndUpdateBatches(testAllocation, true);
+        Set<String> replicaBatches = testShardsBatchGatewayAllocator.createAndUpdateBatches(testAllocation, false);
+        assertEquals(expectedBatchSize, primaryBatches.size());
+        assertEquals(expectedBatchSize, replicaBatches.size());
+        assertEquals(expectedBatchSize, testShardsBatchGatewayAllocator.getBatchIdToStartedShardBatch().size());
+        assertEquals(expectedBatchSize, testShardsBatchGatewayAllocator.getBatchIdToStoreShardBatch().size());
+        assertEquals(testShardsBatchGatewayAllocator.getBatchIdToStartedShardBatch().keySet(), primaryBatches);
+        assertEquals(testShardsBatchGatewayAllocator.getBatchIdToStoreShardBatch().keySet(), replicaBatches);
+        return new Tuple<>(primaryBatches, replicaBatches);
+    }
+}
diff --git a/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java b/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java
index 74bae7b5eb7cf..3ba98c44f8d3e 100644
--- a/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java
+++ b/server/src/test/java/org/opensearch/gateway/GatewayMetaStatePersistedStateTests.java
@@ -71,6 +71,7 @@
 import org.opensearch.gateway.remote.RemotePersistenceStats;
 import org.opensearch.index.recovery.RemoteStoreRestoreService;
 import org.opensearch.index.recovery.RemoteStoreRestoreService.RemoteRestoreResult;
+import org.opensearch.index.remote.RemoteIndexPathUploader;
 import org.opensearch.node.Node;
 import org.opensearch.repositories.RepositoriesService;
 import org.opensearch.repositories.fs.FsRepository;
@@ -473,20 +474,23 @@ public void testDataOnlyNodePersistence() throws Exception {
             );
             Supplier<RemoteClusterStateService> remoteClusterStateServiceSupplier = () -> {
                 if (isRemoteStoreClusterStateEnabled(settings)) {
+                    Supplier<RepositoriesService> repositoriesServiceSupplier = () -> new RepositoriesService(
+                        settings,
+                        clusterService,
+                        transportService,
+                        Collections.emptyMap(),
+                        Collections.emptyMap(),
+                        transportService.getThreadPool()
+                    );
+                    ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
                     return new RemoteClusterStateService(
                         nodeEnvironment.nodeId(),
-                        () -> new RepositoriesService(
-                            settings,
-                            clusterService,
-                            transportService,
-                            Collections.emptyMap(),
-                            Collections.emptyMap(),
-                            transportService.getThreadPool()
-                        ),
+                        repositoriesServiceSupplier,
                         settings,
-                        new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
+                        clusterSettings,
                         () -> 0L,
-                        threadPool
+                        threadPool,
+                        List.of(new RemoteIndexPathUploader(threadPool, settings, repositoriesServiceSupplier, clusterSettings))
                     );
                 } else {
                     return null;
diff --git a/server/src/test/java/org/opensearch/gateway/PrimaryShardBatchAllocatorTests.java b/server/src/test/java/org/opensearch/gateway/PrimaryShardBatchAllocatorTests.java
index 4796def2b8902..522ad2a64ea5d 100644
--- a/server/src/test/java/org/opensearch/gateway/PrimaryShardBatchAllocatorTests.java
+++ b/server/src/test/java/org/opensearch/gateway/PrimaryShardBatchAllocatorTests.java
@@ -29,6 +29,7 @@
 import org.opensearch.common.util.set.Sets;
 import org.opensearch.core.index.shard.ShardId;
 import org.opensearch.env.Environment;
+import org.opensearch.gateway.TransportNodesGatewayStartedShardHelper.GatewayStartedShard;
 import org.opensearch.index.IndexSettings;
 import org.opensearch.index.codec.CodecService;
 import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint;
@@ -289,14 +290,9 @@ public TestBatchAllocator addData(
             if (data == null) {
                 data = new HashMap<>();
             }
-            Map<ShardId, TransportNodesGatewayStartedShardHelper.GatewayStartedShard> shardData = Map.of(
+            Map<ShardId, GatewayStartedShard> shardData = Map.of(
                 shardId,
-                new TransportNodesGatewayStartedShardHelper.GatewayStartedShard(
-                    allocationId,
-                    primary,
-                    replicationCheckpoint,
-                    storeException
-                )
+                new GatewayStartedShard(allocationId, primary, replicationCheckpoint, storeException)
             );
             data.put(node, new TransportNodesListGatewayStartedShardsBatch.NodeGatewayStartedShardsBatch(node, shardData));
             return this;
@@ -313,16 +309,8 @@ public TestBatchAllocator addShardData(
             if (data == null) {
                 data = new HashMap<>();
             }
-            Map<ShardId, TransportNodesGatewayStartedShardHelper.GatewayStartedShard> shardData = new HashMap<>();
-            shardData.put(
-                shardId,
-                new TransportNodesGatewayStartedShardHelper.GatewayStartedShard(
-                    allocationId,
-                    primary,
-                    replicationCheckpoint,
-                    storeException
-                )
-            );
+            Map<ShardId, GatewayStartedShard> shardData = new HashMap<>();
+            shardData.put(shardId, new GatewayStartedShard(allocationId, primary, replicationCheckpoint, storeException));
             if (data.get(node) != null) shardData.putAll(data.get(node).getNodeGatewayStartedShardsBatch());
             data.put(node, new TransportNodesListGatewayStartedShardsBatch.NodeGatewayStartedShardsBatch(node, shardData));
             return this;
diff --git a/server/src/test/java/org/opensearch/gateway/ShardBatchCacheTests.java b/server/src/test/java/org/opensearch/gateway/ShardBatchCacheTests.java
new file mode 100644
index 0000000000000..12030ad41d508
--- /dev/null
+++ b/server/src/test/java/org/opensearch/gateway/ShardBatchCacheTests.java
@@ -0,0 +1,230 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.gateway;
+
+import org.opensearch.cluster.OpenSearchAllocationTestCase;
+import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.cluster.node.DiscoveryNodes;
+import org.opensearch.cluster.routing.ShardRouting;
+import org.opensearch.cluster.routing.ShardRoutingState;
+import org.opensearch.cluster.routing.TestShardRouting;
+import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException;
+import org.opensearch.core.index.shard.ShardId;
+import org.opensearch.gateway.TransportNodesGatewayStartedShardHelper.GatewayStartedShard;
+import org.opensearch.gateway.TransportNodesListGatewayStartedShardsBatch.NodeGatewayStartedShardsBatch;
+import org.opensearch.indices.store.ShardAttributes;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class ShardBatchCacheTests extends OpenSearchAllocationTestCase {
+    private static final String BATCH_ID = "b1";
+    private final DiscoveryNode node1 = newNode("node1");
+    private final DiscoveryNode node2 = newNode("node2");
+    private final Map<ShardId, ShardsBatchGatewayAllocator.ShardEntry> batchInfo = new HashMap<>();
+    private AsyncShardBatchFetch.ShardBatchCache<NodeGatewayStartedShardsBatch, GatewayStartedShard> shardCache;
+    private List<ShardId> shardsInBatch = new ArrayList<>();
+    private static final int NUMBER_OF_SHARDS_DEFAULT = 10;
+
+    private enum ResponseType {
+        NULL,
+        EMPTY,
+        FAILURE,
+        VALID
+    }
+
+    public void setupShardBatchCache(String batchId, int numberOfShards) {
+        Map<ShardId, ShardAttributes> shardAttributesMap = new HashMap<>();
+        fillShards(shardAttributesMap, numberOfShards);
+        this.shardCache = new AsyncShardBatchFetch.ShardBatchCache<>(
+            logger,
+            "batch_shards_started",
+            shardAttributesMap,
+            "BatchID=[" + batchId + "]",
+            GatewayStartedShard.class,
+            new GatewayStartedShard(null, false, null, null),
+            GatewayStartedShard::isEmpty,
+            new ShardBatchResponseFactory<>(true)
+        );
+    }
+
+    public void testClearShardCache() {
+        setupShardBatchCache(BATCH_ID, NUMBER_OF_SHARDS_DEFAULT);
+        ShardId shard = shardsInBatch.iterator().next();
+        this.shardCache.initData(node1);
+        this.shardCache.markAsFetching(List.of(node1.getId()), 1);
+        this.shardCache.putData(node1, new NodeGatewayStartedShardsBatch(node1, getPrimaryResponse(shardsInBatch, ResponseType.EMPTY)));
+        assertTrue(
+            this.shardCache.getCacheData(DiscoveryNodes.builder().add(node1).build(), null)
+                .get(node1)
+                .getNodeGatewayStartedShardsBatch()
+                .containsKey(shard)
+        );
+        this.shardCache.deleteShard(shard);
+        assertFalse(
+            this.shardCache.getCacheData(DiscoveryNodes.builder().add(node1).build(), null)
+                .get(node1)
+                .getNodeGatewayStartedShardsBatch()
+                .containsKey(shard)
+        );
+    }
+
+    public void testGetCacheData() {
+        setupShardBatchCache(BATCH_ID, NUMBER_OF_SHARDS_DEFAULT);
+        ShardId shard = shardsInBatch.iterator().next();
+        this.shardCache.initData(node1);
+        this.shardCache.initData(node2);
+        this.shardCache.markAsFetching(List.of(node1.getId(), node2.getId()), 1);
+        this.shardCache.putData(node1, new NodeGatewayStartedShardsBatch(node1, getPrimaryResponse(shardsInBatch, ResponseType.EMPTY)));
+        assertTrue(
+            this.shardCache.getCacheData(DiscoveryNodes.builder().add(node1).build(), null)
+                .get(node1)
+                .getNodeGatewayStartedShardsBatch()
+                .containsKey(shard)
+        );
+        assertTrue(
+            this.shardCache.getCacheData(DiscoveryNodes.builder().add(node2).build(), null)
+                .get(node2)
+                .getNodeGatewayStartedShardsBatch()
+                .isEmpty()
+        );
+    }
+
+    public void testInitCacheData() {
+        setupShardBatchCache(BATCH_ID, NUMBER_OF_SHARDS_DEFAULT);
+        this.shardCache.initData(node1);
+        this.shardCache.initData(node2);
+        assertEquals(2, shardCache.getCache().size());
+
+        // test getData without fetch
+        assertTrue(shardCache.getData(node1).getNodeGatewayStartedShardsBatch().isEmpty());
+    }
+
+    public void testPutData() {
+        // test empty and non-empty responses
+        setupShardBatchCache(BATCH_ID, NUMBER_OF_SHARDS_DEFAULT);
+        ShardId shard = shardsInBatch.iterator().next();
+        this.shardCache.initData(node1);
+        this.shardCache.initData(node2);
+        this.shardCache.markAsFetching(List.of(node1.getId(), node2.getId()), 1);
+        this.shardCache.putData(node1, new NodeGatewayStartedShardsBatch(node1, getPrimaryResponse(shardsInBatch, ResponseType.VALID)));
+        this.shardCache.putData(node2, new NodeGatewayStartedShardsBatch(node1, getPrimaryResponse(shardsInBatch, ResponseType.EMPTY)));
+
+        // assert that fetching is done as both node's responses are stored in cache
+        assertFalse(this.shardCache.getCache().get(node1.getId()).isFetching());
+        assertFalse(this.shardCache.getCache().get(node2.getId()).isFetching());
+
+        Map<DiscoveryNode, NodeGatewayStartedShardsBatch> fetchData = shardCache.getCacheData(
+            DiscoveryNodes.builder().add(node1).add(node2).build(),
+            null
+        );
+        assertEquals(2, fetchData.size());
+        assertEquals(10, fetchData.get(node1).getNodeGatewayStartedShardsBatch().size());
+        assertEquals("alloc-1", fetchData.get(node1).getNodeGatewayStartedShardsBatch().get(shard).allocationId());
+
+        assertEquals(10, fetchData.get(node2).getNodeGatewayStartedShardsBatch().size());
+        assertTrue(GatewayStartedShard.isEmpty(fetchData.get(node2).getNodeGatewayStartedShardsBatch().get(shard)));
+
+        // test GetData after fetch
+        assertEquals(10, shardCache.getData(node1).getNodeGatewayStartedShardsBatch().size());
+    }
+
+    public void testNullResponses() {
+        setupShardBatchCache(BATCH_ID, NUMBER_OF_SHARDS_DEFAULT);
+        this.shardCache.initData(node1);
+        this.shardCache.markAsFetching(List.of(node1.getId()), 1);
+        this.shardCache.putData(node1, new NodeGatewayStartedShardsBatch(node1, getPrimaryResponse(shardsInBatch, ResponseType.NULL)));
+
+        Map<DiscoveryNode, NodeGatewayStartedShardsBatch> fetchData = shardCache.getCacheData(
+            DiscoveryNodes.builder().add(node1).build(),
+            null
+        );
+        assertTrue(fetchData.get(node1).getNodeGatewayStartedShardsBatch().isEmpty());
+    }
+
+    public void testShardsDataWithException() {
+        setupShardBatchCache(BATCH_ID, NUMBER_OF_SHARDS_DEFAULT);
+        this.shardCache.initData(node1);
+        this.shardCache.initData(node2);
+        this.shardCache.markAsFetching(List.of(node1.getId(), node2.getId()), 1);
+        this.shardCache.putData(node1, new NodeGatewayStartedShardsBatch(node1, getFailedPrimaryResponse(shardsInBatch, 5)));
+        Map<DiscoveryNode, NodeGatewayStartedShardsBatch> fetchData = shardCache.getCacheData(
+            DiscoveryNodes.builder().add(node1).add(node2).build(),
+            null
+        );
+
+        assertEquals(10, batchInfo.size());
+        assertEquals(2, fetchData.size());
+        assertEquals(10, fetchData.get(node1).getNodeGatewayStartedShardsBatch().size());
+        assertTrue(fetchData.get(node2).getNodeGatewayStartedShardsBatch().isEmpty());
+    }
+
+    private Map<ShardId, GatewayStartedShard> getPrimaryResponse(List<ShardId> shards, ResponseType responseType) {
+        int allocationId = 1;
+        Map<ShardId, GatewayStartedShard> shardData = new HashMap<>();
+        for (ShardId shard : shards) {
+            switch (responseType) {
+                case NULL:
+                    shardData.put(shard, null);
+                    break;
+                case EMPTY:
+                    shardData.put(shard, new GatewayStartedShard(null, false, null, null));
+                    break;
+                case VALID:
+                    shardData.put(shard, new GatewayStartedShard("alloc-" + allocationId++, false, null, null));
+                    break;
+                default:
+                    throw new AssertionError("unknown response type");
+            }
+        }
+        return shardData;
+    }
+
+    private Map<ShardId, GatewayStartedShard> getFailedPrimaryResponse(List<ShardId> shards, int failedShardsCount) {
+        int allocationId = 1;
+        Map<ShardId, GatewayStartedShard> shardData = new HashMap<>();
+        for (ShardId shard : shards) {
+            if (failedShardsCount-- > 0) {
+                shardData.put(
+                    shard,
+                    new GatewayStartedShard("alloc-" + allocationId++, false, null, new OpenSearchRejectedExecutionException())
+                );
+            } else {
+                shardData.put(shard, new GatewayStartedShard("alloc-" + allocationId++, false, null, null));
+            }
+        }
+        return shardData;
+    }
+
+    private void fillShards(Map<ShardId, ShardAttributes> shardAttributesMap, int numberOfShards) {
+        shardsInBatch = BatchTestUtil.setUpShards(numberOfShards);
+        for (ShardId shardId : shardsInBatch) {
+            ShardAttributes attr = new ShardAttributes("");
+            shardAttributesMap.put(shardId, attr);
+            batchInfo.put(
+                shardId,
+                new ShardsBatchGatewayAllocator.ShardEntry(attr, randomShardRouting(shardId.getIndexName(), shardId.id()))
+            );
+        }
+    }
+
+    private ShardRouting randomShardRouting(String index, int shard) {
+        ShardRoutingState state = randomFrom(ShardRoutingState.values());
+        return TestShardRouting.newShardRouting(
+            index,
+            shard,
+            state == ShardRoutingState.UNASSIGNED ? null : "1",
+            state == ShardRoutingState.RELOCATING ? "2" : null,
+            state != ShardRoutingState.UNASSIGNED && randomBoolean(),
+            state
+        );
+    }
+}
diff --git a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
index 65477051cdb30..9f321cd62847c 100644
--- a/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
+++ b/server/src/test/java/org/opensearch/gateway/remote/RemoteClusterStateServiceTests.java
@@ -38,6 +38,7 @@
 import org.opensearch.core.index.Index;
 import org.opensearch.core.xcontent.NamedXContentRegistry;
 import org.opensearch.gateway.remote.ClusterMetadataManifest.UploadedIndexMetadata;
+import org.opensearch.index.remote.RemoteIndexPathUploader;
 import org.opensearch.index.remote.RemoteStoreUtils;
 import org.opensearch.indices.IndicesModule;
 import org.opensearch.repositories.FilterRepository;
@@ -154,7 +155,8 @@ public void setup() {
             settings,
             clusterSettings,
             () -> 0L,
-            threadPool
+            threadPool,
+            List.of(new RemoteIndexPathUploader(threadPool, settings, repositoriesServiceSupplier, clusterSettings))
         );
     }
 
@@ -173,15 +175,17 @@ public void testFailWriteFullMetadataNonClusterManagerNode() throws IOException
 
     public void testFailInitializationWhenRemoteStateDisabled() {
         final Settings settings = Settings.builder().build();
+        ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
         assertThrows(
             AssertionError.class,
             () -> new RemoteClusterStateService(
                 "test-node-id",
                 repositoriesServiceSupplier,
                 settings,
-                new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
+                clusterSettings,
                 () -> 0L,
-                threadPool
+                threadPool,
+                List.of(new RemoteIndexPathUploader(threadPool, settings, repositoriesServiceSupplier, clusterSettings))
             )
         );
     }
diff --git a/server/src/test/java/org/opensearch/index/IndexModuleTests.java b/server/src/test/java/org/opensearch/index/IndexModuleTests.java
index 97bc822be7d51..4ce4936c047d9 100644
--- a/server/src/test/java/org/opensearch/index/IndexModuleTests.java
+++ b/server/src/test/java/org/opensearch/index/IndexModuleTests.java
@@ -99,6 +99,7 @@
 import org.opensearch.index.translog.InternalTranslogFactory;
 import org.opensearch.index.translog.RemoteBlobStoreInternalTranslogFactory;
 import org.opensearch.index.translog.TranslogFactory;
+import org.opensearch.indices.DefaultRemoteStoreSettings;
 import org.opensearch.indices.IndicesModule;
 import org.opensearch.indices.IndicesQueryCache;
 import org.opensearch.indices.analysis.AnalysisModule;
@@ -236,7 +237,8 @@ private IndexService newIndexService(IndexModule module) throws IOException {
                     repositoriesServiceReference::get,
                     threadPool,
                     indexSettings.getRemoteStoreTranslogRepository(),
-                    new RemoteTranslogTransferTracker(shardRouting.shardId(), 10)
+                    new RemoteTranslogTransferTracker(shardRouting.shardId(), 10),
+                    DefaultRemoteStoreSettings.INSTANCE
                 );
             }
             return new InternalTranslogFactory();
@@ -261,8 +263,8 @@ private IndexService newIndexService(IndexModule module) throws IOException {
             new RemoteSegmentStoreDirectoryFactory(() -> repositoriesService, threadPool),
             translogFactorySupplier,
             () -> IndexSettings.DEFAULT_REFRESH_INTERVAL,
-            () -> IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL,
-            DefaultRecoverySettings.INSTANCE
+            DefaultRecoverySettings.INSTANCE,
+            DefaultRemoteStoreSettings.INSTANCE
         );
     }
 
diff --git a/server/src/test/java/org/opensearch/index/IndexSettingsTests.java b/server/src/test/java/org/opensearch/index/IndexSettingsTests.java
index e4ce879a5ec5e..474ec73d5fe61 100644
--- a/server/src/test/java/org/opensearch/index/IndexSettingsTests.java
+++ b/server/src/test/java/org/opensearch/index/IndexSettingsTests.java
@@ -1053,4 +1053,15 @@ public void testDefaultSearchPipeline() throws Exception {
         settings.updateIndexMetadata(metadata);
         assertEquals("foo", settings.getDefaultSearchPipeline());
     }
+
+    public void testIsOnRemoteNode() {
+        Version version = VersionUtils.getPreviousVersion();
+        Settings theSettings = Settings.builder()
+            .put(IndexMetadata.SETTING_VERSION_CREATED, version)
+            .put(IndexMetadata.SETTING_INDEX_UUID, "0xdeadbeef")
+            .build();
+        Settings nodeSettings = Settings.builder().put("node.attr.remote_store.translog.repository", "my-repo-1").build();
+        IndexSettings settings = newIndexSettings(newIndexMeta("index", theSettings), nodeSettings);
+        assertTrue("Index should be on remote node", settings.isAssignedOnRemoteNode());
+    }
 }
diff --git a/server/src/test/java/org/opensearch/index/codec/fuzzy/FuzzyFilterPostingsFormatTests.java b/server/src/test/java/org/opensearch/index/codec/fuzzy/FuzzyFilterPostingsFormatTests.java
index 868c2175d0689..c92ee191193c6 100644
--- a/server/src/test/java/org/opensearch/index/codec/fuzzy/FuzzyFilterPostingsFormatTests.java
+++ b/server/src/test/java/org/opensearch/index/codec/fuzzy/FuzzyFilterPostingsFormatTests.java
@@ -10,10 +10,12 @@
 
 import org.apache.lucene.codecs.Codec;
 import org.apache.lucene.tests.index.BasePostingsFormatTestCase;
+import org.apache.lucene.tests.util.LuceneTestCase;
 import org.apache.lucene.tests.util.TestUtil;
 
 import java.util.TreeMap;
 
+@LuceneTestCase.SuppressSysoutChecks(bugUrl = "we log a lot on purpose")
 public class FuzzyFilterPostingsFormatTests extends BasePostingsFormatTestCase {
 
     private TreeMap<String, FuzzySetParameters> params = new TreeMap<>() {
diff --git a/server/src/test/java/org/opensearch/index/engine/EngineConfigTests.java b/server/src/test/java/org/opensearch/index/engine/EngineConfigTests.java
index 9197b959a54f1..f42188d55bca3 100644
--- a/server/src/test/java/org/opensearch/index/engine/EngineConfigTests.java
+++ b/server/src/test/java/org/opensearch/index/engine/EngineConfigTests.java
@@ -32,6 +32,13 @@ public void setUp() throws Exception {
         defaultIndexSettings = IndexSettingsModule.newIndexSettings("test", defaultIndexMetadata.getSettings());
     }
 
+    public void testEngineConfig_DefaultValueFoUseCompoundFile() {
+        EngineConfig config = new EngineConfig.Builder().indexSettings(defaultIndexSettings)
+            .retentionLeasesSupplier(() -> RetentionLeases.EMPTY)
+            .build();
+        assertTrue(config.useCompoundFile());
+    }
+
     public void testEngineConfig_DefaultValueForReadOnlyEngine() {
         EngineConfig config = new EngineConfig.Builder().indexSettings(defaultIndexSettings)
             .retentionLeasesSupplier(() -> RetentionLeases.EMPTY)
diff --git a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java
index cc927a19fd01a..54a562642d4ab 100644
--- a/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java
+++ b/server/src/test/java/org/opensearch/index/engine/InternalEngineTests.java
@@ -342,6 +342,58 @@ public void testVerboseSegments() throws Exception {
         }
     }
 
+    public void testSegmentsWithUseCompoundFileFlag_true() throws IOException {
+        try (Store store = createStore(); Engine engine = createEngine(defaultSettings, store, createTempDir(), new TieredMergePolicy())) {
+            ParsedDocument doc = testParsedDocument("1", null, testDocument(), B_1, null);
+            Engine.Index index = indexForDoc(doc);
+            engine.index(index);
+            engine.flush();
+            final List<Segment> segments = engine.segments(false);
+            assertThat(segments, hasSize(1));
+            assertTrue(segments.get(0).compound);
+            boolean cfeCompoundFileFound = false;
+            boolean cfsCompoundFileFound = false;
+            for (final String fileName : store.readLastCommittedSegmentsInfo().files(true)) {
+                if (fileName.endsWith(".cfe")) {
+                    cfeCompoundFileFound = true;
+                }
+                if (fileName.endsWith(".cfs")) {
+                    cfsCompoundFileFound = true;
+                }
+            }
+            Assert.assertTrue(cfeCompoundFileFound);
+            Assert.assertTrue(cfsCompoundFileFound);
+        }
+    }
+
+    public void testSegmentsWithUseCompoundFileFlag_false() throws IOException {
+        final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(
+            "test",
+            Settings.builder().put(defaultSettings.getSettings()).put(EngineConfig.INDEX_USE_COMPOUND_FILE.getKey(), false).build()
+        );
+        try (Store store = createStore(); Engine engine = createEngine(indexSettings, store, createTempDir(), new TieredMergePolicy())) {
+            ParsedDocument doc = testParsedDocument("1", null, testDocument(), B_1, null);
+            Engine.Index index = indexForDoc(doc);
+            engine.index(index);
+            engine.flush();
+            final List<Segment> segments = engine.segments(false);
+            assertThat(segments, hasSize(1));
+            assertFalse(segments.get(0).compound);
+            boolean cfeCompoundFileFound = false;
+            boolean cfsCompoundFileFound = false;
+            for (final String fileName : store.readLastCommittedSegmentsInfo().files(true)) {
+                if (fileName.endsWith(".cfe")) {
+                    cfeCompoundFileFound = true;
+                }
+                if (fileName.endsWith(".cfs")) {
+                    cfsCompoundFileFound = true;
+                }
+            }
+            Assert.assertFalse(cfeCompoundFileFound);
+            Assert.assertFalse(cfsCompoundFileFound);
+        }
+    }
+
     public void testSegmentsWithMergeFlag() throws Exception {
         try (Store store = createStore(); Engine engine = createEngine(defaultSettings, store, createTempDir(), new TieredMergePolicy())) {
             ParsedDocument doc = testParsedDocument("1", null, testDocument(), B_1, null);
@@ -4002,7 +4054,7 @@ public void testRecoverFromForeignTranslog() throws IOException {
         final Path badTranslogLog = createTempDir();
         final String badUUID = Translog.createEmptyTranslog(badTranslogLog, SequenceNumbers.NO_OPS_PERFORMED, shardId, primaryTerm.get());
         Translog translog = new LocalTranslog(
-            new TranslogConfig(shardId, badTranslogLog, INDEX_SETTINGS, BigArrays.NON_RECYCLING_INSTANCE, ""),
+            new TranslogConfig(shardId, badTranslogLog, INDEX_SETTINGS, BigArrays.NON_RECYCLING_INSTANCE, "", false),
             badUUID,
             createTranslogDeletionPolicy(INDEX_SETTINGS),
             () -> SequenceNumbers.NO_OPS_PERFORMED,
@@ -4020,7 +4072,8 @@ public void testRecoverFromForeignTranslog() throws IOException {
             translog.location(),
             config.getIndexSettings(),
             BigArrays.NON_RECYCLING_INSTANCE,
-            ""
+            "",
+            false
         );
 
         EngineConfig brokenConfig = new EngineConfig.Builder().shardId(shardId)
@@ -7714,7 +7767,8 @@ public void testNotWarmUpSearcherInEngineCtor() throws Exception {
                 createTempDir(),
                 config.getTranslogConfig().getIndexSettings(),
                 config.getTranslogConfig().getBigArrays(),
-                ""
+                "",
+                false
             );
             EngineConfig configWithWarmer = new EngineConfig.Builder().shardId(config.getShardId())
                 .threadPool(config.getThreadPool())
diff --git a/server/src/test/java/org/opensearch/index/mapper/BooleanFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/BooleanFieldMapperTests.java
index 8dec03a353d16..5392bd6c358d3 100644
--- a/server/src/test/java/org/opensearch/index/mapper/BooleanFieldMapperTests.java
+++ b/server/src/test/java/org/opensearch/index/mapper/BooleanFieldMapperTests.java
@@ -209,4 +209,29 @@ public void testBoosts() throws Exception {
         assertEquals(new BoostQuery(new TermQuery(new Term("field", "T")), 2.0f), ft.termQuery("true", null));
         assertParseMaximalWarnings();
     }
+
+    public void testIndexedValueForSearch() throws Exception {
+        assertEquals(new BooleanFieldMapper.BooleanFieldType("bool").indexedValueForSearch(null), BooleanFieldMapper.Values.FALSE);
+
+        assertEquals(new BooleanFieldMapper.BooleanFieldType("bool").indexedValueForSearch(false), BooleanFieldMapper.Values.FALSE);
+
+        assertEquals(new BooleanFieldMapper.BooleanFieldType("bool").indexedValueForSearch(true), BooleanFieldMapper.Values.TRUE);
+
+        assertEquals(
+            new BooleanFieldMapper.BooleanFieldType("bool").indexedValueForSearch(new BytesRef("true")),
+            BooleanFieldMapper.Values.TRUE
+        );
+
+        assertEquals(
+            new BooleanFieldMapper.BooleanFieldType("bool").indexedValueForSearch(new BytesRef("false")),
+            BooleanFieldMapper.Values.FALSE
+        );
+
+        IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
+            () -> new BooleanFieldMapper.BooleanFieldType("bool").indexedValueForSearch(new BytesRef("random"))
+        );
+
+        assertEquals("Can't parse boolean value [random], expected [true] or [false]", e.getMessage());
+    }
 }
diff --git a/server/src/test/java/org/opensearch/index/mapper/BooleanFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/BooleanFieldTypeTests.java
index 14092706411cb..aab63fc30efd7 100644
--- a/server/src/test/java/org/opensearch/index/mapper/BooleanFieldTypeTests.java
+++ b/server/src/test/java/org/opensearch/index/mapper/BooleanFieldTypeTests.java
@@ -31,11 +31,18 @@
 
 package org.opensearch.index.mapper;
 
+import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BoostQuery;
+import org.apache.lucene.search.DocValuesFieldExistsQuery;
+import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.util.BytesRef;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Collections;
+import java.util.List;
 
 public class BooleanFieldTypeTests extends FieldTypeTestCase {
 
@@ -59,9 +66,67 @@ public void testTermQuery() {
         assertEquals(new TermQuery(new Term("field", "T")), ft.termQuery("true", null));
         assertEquals(new TermQuery(new Term("field", "F")), ft.termQuery("false", null));
 
-        MappedFieldType unsearchable = new BooleanFieldMapper.BooleanFieldType("field", false);
+        MappedFieldType doc_ft = new BooleanFieldMapper.BooleanFieldType("field", false, true);
+        assertEquals(SortedNumericDocValuesField.newSlowExactQuery("field", 1), doc_ft.termQuery("true", null));
+        assertEquals(SortedNumericDocValuesField.newSlowExactQuery("field", 0), doc_ft.termQuery("false", null));
+
+        MappedFieldType boost_ft = new BooleanFieldMapper.BooleanFieldType("field");
+        boost_ft.setBoost(2f);
+        assertEquals(new BoostQuery(new TermQuery(new Term("field", "T")), 2f), boost_ft.termQuery("true", null));
+        assertEquals(new BoostQuery(new TermQuery(new Term("field", "F")), 2f), boost_ft.termQuery("false", null));
+
+        MappedFieldType unsearchable = new BooleanFieldMapper.BooleanFieldType("field", false, false);
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery("true", null));
-        assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage());
+        assertEquals("Cannot search on field [field] since it is both not indexed, and does not have doc_values enabled.", e.getMessage());
+    }
+
+    public void testTermsQuery() {
+        MappedFieldType ft = new BooleanFieldMapper.BooleanFieldType("field");
+        List<BytesRef> terms = new ArrayList<>();
+        terms.add(new BytesRef("true"));
+        terms.add(new BytesRef("false"));
+        assertEquals(new DocValuesFieldExistsQuery("field"), ft.termsQuery(terms, null));
+
+        List<BytesRef> newTerms = new ArrayList<>();
+        newTerms.add(new BytesRef("true"));
+        assertEquals(new TermQuery(new Term("field", "T")), ft.termsQuery(newTerms, null));
+
+        List<BytesRef> incorrectTerms = new ArrayList<>();
+        incorrectTerms.add(new BytesRef("true"));
+        incorrectTerms.add(new BytesRef("random"));
+        IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> ft.termsQuery(incorrectTerms, null));
+        assertEquals("Can't parse boolean value [random], expected [true] or [false]", ex.getMessage());
+
+        MappedFieldType doc_only_ft = new BooleanFieldMapper.BooleanFieldType("field", false, true);
+
+        assertEquals(SortedNumericDocValuesField.newSlowExactQuery("field", 1), doc_only_ft.termsQuery(newTerms, null));
+
+        MappedFieldType unsearchable = new BooleanFieldMapper.BooleanFieldType("field", false, false);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termsQuery(terms, null));
+        assertEquals("Cannot search on field [field] since it is both not indexed, and does not have doc_values enabled.", e.getMessage());
+    }
+
+    public void testRangeQuery() {
+        BooleanFieldMapper.BooleanFieldType ft = new BooleanFieldMapper.BooleanFieldType("field");
+        assertEquals(new DocValuesFieldExistsQuery("field"), ft.rangeQuery(false, true, true, true, null));
+
+        assertEquals(new TermQuery(new Term("field", "T")), ft.rangeQuery(false, true, false, true, null));
+
+        assertEquals(new TermQuery(new Term("field", "F")), ft.rangeQuery(false, true, true, false, null));
+
+        assertEquals(new MatchNoDocsQuery(), ft.rangeQuery(false, true, false, false, null));
+
+        assertEquals(new MatchNoDocsQuery(), ft.rangeQuery(false, true, false, false, null));
+
+        assertEquals(new TermQuery(new Term("field", "F")), ft.rangeQuery(false, false, true, true, null));
+
+        assertEquals(new TermQuery(new Term("field", "F")), ft.rangeQuery(null, false, true, true, null));
+
+        assertEquals(new DocValuesFieldExistsQuery("field"), ft.rangeQuery(false, null, true, true, null));
+
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> ft.rangeQuery("random", null, true, true, null));
+
+        assertEquals("Can't parse boolean value [random], expected [true] or [false]", e.getMessage());
     }
 
     public void testFetchSourceValue() throws IOException {
diff --git a/server/src/test/java/org/opensearch/index/mapper/ConstantKeywordFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/ConstantKeywordFieldMapperTests.java
new file mode 100644
index 0000000000000..65dd3b6447663
--- /dev/null
+++ b/server/src/test/java/org/opensearch/index/mapper/ConstantKeywordFieldMapperTests.java
@@ -0,0 +1,114 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.mapper;
+
+import org.apache.lucene.index.IndexableField;
+import org.opensearch.OpenSearchParseException;
+import org.opensearch.common.CheckedConsumer;
+import org.opensearch.common.compress.CompressedXContent;
+import org.opensearch.common.xcontent.XContentFactory;
+import org.opensearch.common.xcontent.json.JsonXContent;
+import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
+import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.index.IndexService;
+import org.opensearch.plugins.Plugin;
+import org.opensearch.test.InternalSettingsPlugin;
+import org.opensearch.test.OpenSearchSingleNodeTestCase;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.Collection;
+
+import static org.hamcrest.Matchers.containsString;
+
+public class ConstantKeywordFieldMapperTests extends OpenSearchSingleNodeTestCase {
+
+    private IndexService indexService;
+    private DocumentMapperParser parser;
+
+    @Override
+    protected Collection<Class<? extends Plugin>> getPlugins() {
+        return pluginList(InternalSettingsPlugin.class);
+    }
+
+    @Before
+    public void setup() {
+        indexService = createIndex("test");
+        parser = indexService.mapperService().documentMapperParser();
+    }
+
+    public void testDefaultDisabledIndexMapper() throws Exception {
+
+        XContentBuilder mapping = XContentFactory.jsonBuilder()
+            .startObject()
+            .startObject("type")
+            .startObject("properties")
+            .startObject("field")
+            .field("type", "constant_keyword")
+            .field("value", "default_value")
+            .endObject()
+            .startObject("field2")
+            .field("type", "keyword")
+            .endObject();
+        mapping = mapping.endObject().endObject().endObject();
+        DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping.toString()));
+
+        MapperParsingException e = expectThrows(MapperParsingException.class, () -> mapper.parse(source(b -> {
+            b.field("field", "sdf");
+            b.field("field2", "szdfvsddf");
+        })));
+        assertThat(
+            e.getMessage(),
+            containsString(
+                "failed to parse field [field] of type [constant_keyword] in document with id '1'. Preview of field's value: 'sdf'"
+            )
+        );
+
+        final ParsedDocument doc = mapper.parse(source(b -> {
+            b.field("field", "default_value");
+            b.field("field2", "field_2_value");
+        }));
+
+        final IndexableField field = doc.rootDoc().getField("field");
+
+        // constantKeywordField should not be stored
+        assertNull(field);
+    }
+
+    public void testMissingDefaultIndexMapper() throws Exception {
+
+        final XContentBuilder mapping = XContentFactory.jsonBuilder()
+            .startObject()
+            .startObject("type")
+            .startObject("properties")
+            .startObject("field")
+            .field("type", "constant_keyword")
+            .endObject()
+            .startObject("field2")
+            .field("type", "keyword")
+            .endObject()
+            .endObject()
+            .endObject()
+            .endObject();
+
+        OpenSearchParseException e = expectThrows(
+            OpenSearchParseException.class,
+            () -> parser.parse("type", new CompressedXContent(mapping.toString()))
+        );
+        assertThat(e.getMessage(), containsString("Field [field] is missing required parameter [value]"));
+    }
+
+    private final SourceToParse source(CheckedConsumer<XContentBuilder, IOException> build) throws IOException {
+        XContentBuilder builder = JsonXContent.contentBuilder().startObject();
+        build.accept(builder);
+        builder.endObject();
+        return new SourceToParse("test", "1", BytesReference.bytes(builder), MediaTypeRegistry.JSON);
+    }
+}
diff --git a/server/src/test/java/org/opensearch/index/mapper/ConstantKeywordFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/ConstantKeywordFieldTypeTests.java
new file mode 100644
index 0000000000000..235811539a299
--- /dev/null
+++ b/server/src/test/java/org/opensearch/index/mapper/ConstantKeywordFieldTypeTests.java
@@ -0,0 +1,93 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.mapper;
+
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.MatchNoDocsQuery;
+import org.opensearch.Version;
+import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.common.regex.Regex;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.index.IndexSettings;
+import org.opensearch.index.query.QueryShardContext;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.function.Predicate;
+
+public class ConstantKeywordFieldTypeTests extends FieldTypeTestCase {
+
+    final ConstantKeywordFieldMapper.ConstantKeywordFieldType ft = new ConstantKeywordFieldMapper.ConstantKeywordFieldType(
+        "field",
+        "default"
+    );
+
+    public void testTermQuery() {
+        assertEquals(new MatchAllDocsQuery(), ft.termQuery("default", createContext()));
+        assertEquals(new MatchNoDocsQuery(), ft.termQuery("not_default", createContext()));
+    }
+
+    public void testTermsQuery() {
+        assertEquals(new MatchAllDocsQuery(), ft.termsQuery(Arrays.asList("default", "not_default"), createContext()));
+        assertEquals(new MatchNoDocsQuery(), ft.termsQuery(Arrays.asList("no_default", "not_default"), createContext()));
+        assertEquals(new MatchNoDocsQuery(), ft.termsQuery(List.of(), createContext()));
+    }
+
+    public void testInsensitiveTermQuery() {
+        assertEquals(new MatchAllDocsQuery(), ft.termQueryCaseInsensitive("defaUlt", createContext()));
+        assertEquals(new MatchNoDocsQuery(), ft.termQueryCaseInsensitive("not_defaUlt", createContext()));
+    }
+
+    public void testPrefixQuery() {
+        assertEquals(new MatchAllDocsQuery(), ft.prefixQuery("defau", null, createContext()));
+        assertEquals(new MatchNoDocsQuery(), ft.prefixQuery("not_default", null, createContext()));
+    }
+
+    public void testWildcardQuery() {
+        assertEquals(new MatchAllDocsQuery(), ft.wildcardQuery("defa*lt", null, createContext()));
+        assertEquals(new MatchNoDocsQuery(), ft.wildcardQuery("no_defa*lt", null, createContext()));
+        assertEquals(new MatchAllDocsQuery(), ft.wildcardQuery("defa*", null, createContext()));
+        assertEquals(new MatchAllDocsQuery(), ft.wildcardQuery("*ult", null, createContext()));
+
+    }
+
+    public void testExistsQuery() {
+        assertEquals(new MatchAllDocsQuery(), ft.existsQuery(createContext()));
+    }
+
+    private QueryShardContext createContext() {
+        IndexMetadata indexMetadata = IndexMetadata.builder("index")
+            .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT))
+            .numberOfShards(1)
+            .numberOfReplicas(0)
+            .build();
+        IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY);
+
+        Predicate<String> indexNameMatcher = pattern -> Regex.simpleMatch(pattern, "index");
+        return new QueryShardContext(
+            0,
+            indexSettings,
+            null,
+            null,
+            null,
+            null,
+            null,
+            null,
+            xContentRegistry(),
+            writableRegistry(),
+            null,
+            null,
+            System::currentTimeMillis,
+            null,
+            indexNameMatcher,
+            () -> true,
+            null
+        );
+    }
+}
diff --git a/server/src/test/java/org/opensearch/index/mapper/DateFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/DateFieldMapperTests.java
index 054d3956596af..2aa310ae959d9 100644
--- a/server/src/test/java/org/opensearch/index/mapper/DateFieldMapperTests.java
+++ b/server/src/test/java/org/opensearch/index/mapper/DateFieldMapperTests.java
@@ -35,6 +35,7 @@
 import org.apache.lucene.index.DocValuesType;
 import org.apache.lucene.index.IndexableField;
 import org.opensearch.common.time.DateFormatter;
+import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.index.termvectors.TermVectorsService;
 import org.opensearch.search.DocValueFormat;
@@ -45,8 +46,10 @@
 import java.time.ZonedDateTime;
 import java.util.List;
 
+import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.notNullValue;
+import static org.junit.Assume.assumeThat;
 
 public class DateFieldMapperTests extends MapperTestCase {
 
@@ -146,7 +149,22 @@ public void testStore() throws Exception {
         assertEquals(1457654400000L, storedField.numericValue().longValue());
     }
 
+    public void testIgnoreMalformedLegacy() throws IOException {
+        assumeThat("Using legacy datetime format as default", FeatureFlags.isEnabled(FeatureFlags.DATETIME_FORMATTER_CACHING), is(false));
+        testIgnoreMalformedForValue(
+            "2016-03-99",
+            "failed to parse date field [2016-03-99] with format [strict_date_optional_time||epoch_millis]"
+        );
+        testIgnoreMalformedForValue("-2147483648", "Invalid value for Year (valid values -999999999 - 999999999): -2147483648");
+        testIgnoreMalformedForValue("-522000000", "long overflow");
+    }
+
     public void testIgnoreMalformed() throws IOException {
+        assumeThat(
+            "Using experimental datetime format as default",
+            FeatureFlags.isEnabled(FeatureFlags.DATETIME_FORMATTER_CACHING),
+            is(true)
+        );
         testIgnoreMalformedForValue(
             "2016-03-99",
             "failed to parse date field [2016-03-99] with format [strict_date_time_no_millis||strict_date_optional_time||epoch_millis]"
diff --git a/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java
index ab53ae81ab0ce..db5e1e419de93 100644
--- a/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java
+++ b/server/src/test/java/org/opensearch/index/mapper/DateFieldTypeTests.java
@@ -216,14 +216,14 @@ public void testTermQuery() {
             "field",
             false,
             false,
-            true,
+            false,
             DateFieldMapper.getDefaultDateTimeFormatter(),
             Resolution.MILLISECONDS,
             null,
             Collections.emptyMap()
         );
         IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery(date, context));
-        assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage());
+        assertEquals("Cannot search on field [field] since it is both not indexed, and does not have doc_values enabled.", e.getMessage());
     }
 
     public void testRangeQuery() throws IOException {
@@ -279,7 +279,7 @@ public void testRangeQuery() throws IOException {
             "field",
             false,
             false,
-            true,
+            false,
             DateFieldMapper.getDefaultDateTimeFormatter(),
             Resolution.MILLISECONDS,
             null,
@@ -289,7 +289,7 @@ public void testRangeQuery() throws IOException {
             IllegalArgumentException.class,
             () -> unsearchable.rangeQuery(date1, date2, true, true, null, null, null, context)
         );
-        assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage());
+        assertEquals("Cannot search on field [field] since it is both not indexed, and does not have doc_values enabled.", e.getMessage());
     }
 
     public void testRangeQueryWithIndexSort() {
diff --git a/server/src/test/java/org/opensearch/index/mapper/DerivedFieldMapperQueryTests.java b/server/src/test/java/org/opensearch/index/mapper/DerivedFieldMapperQueryTests.java
new file mode 100644
index 0000000000000..1307028dd27b0
--- /dev/null
+++ b/server/src/test/java/org/opensearch/index/mapper/DerivedFieldMapperQueryTests.java
@@ -0,0 +1,305 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.mapper;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.store.Directory;
+import org.opensearch.common.collect.Tuple;
+import org.opensearch.common.lucene.Lucene;
+import org.opensearch.core.index.Index;
+import org.opensearch.geometry.Rectangle;
+import org.opensearch.index.query.QueryBuilders;
+import org.opensearch.index.query.QueryShardContext;
+import org.opensearch.script.DerivedFieldScript;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+import org.mockito.Mockito;
+
+import static org.opensearch.index.query.QueryBuilders.geoShapeQuery;
+import static org.mockito.Mockito.when;
+
+public class DerivedFieldMapperQueryTests extends MapperServiceTestCase {
+
+    // First element is the document ingested, other elements corresponds to value returned against a given derived field script
+    // Raw Message, Request Succeeded (boolean), Timestamp (long), Client IP, Method, Request Size (double), Duration (long)
+    private static final Object[][] raw_requests = new Object[][] {
+        {
+            "40.135.0.0 GET /images/hm_bg.jpg?size=1.5KB loc 10.0 20.0 HTTP/1.0 200 2024-03-20T08:30:45 1500",
+            true,
+            1710923445000L,
+            "40.135.0.0",
+            "GET",
+            1.5,
+            1500L,
+            new Tuple<>(10.0, 20.0) },
+        {
+            "232.0.0.0 GET /images/hm_bg.jpg?size=2.3KB HTTP/1.0 400 2024-03-20T09:15:20 2300",
+            false,
+            1710926120000L,
+            "232.0.0.0",
+            "GET",
+            2.3,
+            2300L,
+            new Tuple<>(20.0, 30.0) },
+        {
+            "26.1.0.0 DELETE /images/hm_bg.jpg?size=3.7KB HTTP/1.0 200 2024-03-20T10:05:55 3700",
+            true,
+            1710929155000L,
+            "26.1.0.0",
+            "DELETE",
+            3.7,
+            3700L,
+            new Tuple<>(30.0, 40.0) },
+        {
+            "247.37.0.0 GET /french/splash_inet.html?size=4.1KB HTTP/1.0 400 2024-03-20T11:20:10 4100",
+            false,
+            1710933610000L,
+            "247.37.0.0",
+            "GET",
+            4.1,
+            4100L,
+            new Tuple<>(40.0, 50.0) },
+        {
+            "247.37.0.0 DELETE /french/splash_inet.html?size=5.8KB HTTP/1.0 400 2024-03-20T12:45:30 5800",
+            false,
+            1710938730000L,
+            "247.37.0.0",
+            "DELETE",
+            5.8,
+            5800L,
+            new Tuple<>(50.0, 60.0) },
+        {
+            "10.20.30.40 GET /path/to/resource?size=6.3KB HTTP/1.0 200 2024-03-20T13:10:15 6300",
+            true,
+            1710940215000L,
+            "10.20.30.40",
+            "GET",
+            6.3,
+            6300L,
+            new Tuple<>(60.0, 70.0) },
+        {
+            "50.60.70.80 GET /path/to/resource?size=7.2KB HTTP/1.0 404 2024-03-20T14:20:50 7200",
+            false,
+            1710944450000L,
+            "50.60.70.80",
+            "GET",
+            7.2,
+            7200L,
+            new Tuple<>(70.0, 80.0) },
+        {
+            "127.0.0.1 PUT /path/to/resource?size=8.9KB HTTP/1.0 500 2024-03-20T15:30:25 8900",
+            false,
+            1710948625000L,
+            "127.0.0.1",
+            "PUT",
+            8.9,
+            8900L,
+            new Tuple<>(80.0, 90.0) },
+        {
+            "127.0.0.1 GET /path/to/resource?size=9.4KB HTTP/1.0 200 2024-03-20T16:40:15 9400",
+            true,
+            1710952815000L,
+            "127.0.0.1",
+            "GET",
+            9.4,
+            9400L,
+            new Tuple<>(85.0, 90.0) },
+        {
+            "192.168.1.1 GET /path/to/resource?size=10.7KB HTTP/1.0 400 2024-03-20T17:50:40 10700",
+            false,
+            1710957040000L,
+            "192.168.1.1",
+            "GET",
+            10.7,
+            10700L,
+            new Tuple<>(90.0, 90.0) } };
+
+    public void testAllPossibleQueriesOnDerivedFields() throws IOException {
+        MapperService mapperService = createMapperService(topMapping(b -> {
+            b.startObject("properties");
+            {
+                b.startObject("raw_message");
+                {
+                    b.field("type", "text");
+                }
+                b.endObject();
+            }
+            b.endObject();
+            b.startObject("derived");
+            {
+                b.startObject("request_succeeded");
+                {
+                    b.field("type", "boolean");
+                    b.field("script", "");
+                }
+                b.endObject();
+                b.startObject("@timestamp");
+                {
+                    b.field("type", "date");
+                    b.field("script", "");
+                }
+                b.endObject();
+                b.startObject("client_ip");
+                {
+                    b.field("type", "ip");
+                    b.field("script", "");
+                }
+                b.endObject();
+                b.startObject("method");
+                {
+                    b.field("type", "keyword");
+                    b.field("script", "");
+                }
+                b.endObject();
+                b.startObject("request_size");
+                {
+                    b.field("type", "double");
+                    b.field("script", "");
+                }
+                b.endObject();
+                b.startObject("duration");
+                {
+                    b.field("type", "long");
+                    b.field("script", "");
+                }
+                b.endObject();
+                b.startObject("geopoint");
+                {
+                    b.field("type", "geo_point");
+                    b.field("script", "");
+                }
+                b.endObject();
+            }
+            b.endObject();
+        }));
+
+        List<Document> docs = new ArrayList<>();
+        for (Object[] request : raw_requests) {
+            Document document = new Document();
+            document.add(new TextField("raw_message", (String) request[0], Field.Store.YES));
+            docs.add(document);
+        }
+
+        int[] scriptIndex = { 1 };
+
+        // Mock DerivedFieldScript.Factory
+        DerivedFieldScript.Factory factory = (params, lookup) -> (DerivedFieldScript.LeafFactory) ctx -> new DerivedFieldScript(
+            params,
+            lookup,
+            ctx
+        ) {
+            int docId = 0;
+
+            @Override
+            public void setDocument(int docId) {
+                super.setDocument(docId);
+                this.docId = docId;
+            }
+
+            @Override
+            public void execute() {
+                addEmittedValue(raw_requests[docId][scriptIndex[0]]);
+            }
+        };
+
+        QueryShardContext queryShardContext = createQueryShardContext(mapperService);
+        when(queryShardContext.compile(Mockito.any(), Mockito.any())).thenReturn(factory);
+        when(queryShardContext.sourcePath("raw_message")).thenReturn(Set.of("raw_message"));
+        when(queryShardContext.index()).thenReturn(new Index("test_index", "uuid"));
+
+        // Index and Search
+        try (Directory dir = newDirectory()) {
+            IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER));
+            for (Document d : docs) {
+                iw.addDocument(d);
+            }
+            try (IndexReader reader = DirectoryReader.open(iw)) {
+                iw.close();
+
+                IndexSearcher searcher = new IndexSearcher(reader);
+                Query query = QueryBuilders.termQuery("request_succeeded", "true").toQuery(queryShardContext);
+                TopDocs topDocs = searcher.search(query, 10);
+                assertEquals(4, topDocs.totalHits.value);
+
+                // IP Field Term Query
+                scriptIndex[0] = 3;
+                query = QueryBuilders.termQuery("client_ip", "192.168.0.0/16").toQuery(queryShardContext);
+                topDocs = searcher.search(query, 10);
+                assertEquals(1, topDocs.totalHits.value);
+
+                scriptIndex[0] = 4;
+                query = QueryBuilders.termsQuery("method", "DELETE", "PUT").toQuery(queryShardContext);
+                topDocs = searcher.search(query, 10);
+                assertEquals(3, topDocs.totalHits.value);
+
+                query = QueryBuilders.termsQuery("method", "delete").toQuery(queryShardContext);
+                topDocs = searcher.search(query, 10);
+                assertEquals(0, topDocs.totalHits.value);
+
+                query = QueryBuilders.termQuery("method", "delete").caseInsensitive(true).toQuery(queryShardContext);
+                topDocs = searcher.search(query, 10);
+                assertEquals(2, topDocs.totalHits.value);
+
+                // Range queries of types - date, long and double
+                scriptIndex[0] = 2;
+                query = QueryBuilders.rangeQuery("@timestamp").from("2024-03-20T14:20:50").toQuery(queryShardContext);
+                topDocs = searcher.search(query, 10);
+                assertEquals(4, topDocs.totalHits.value);
+
+                scriptIndex[0] = 5;
+                query = QueryBuilders.rangeQuery("request_size").from("4.1").toQuery(queryShardContext);
+                topDocs = searcher.search(query, 10);
+                assertEquals(7, topDocs.totalHits.value);
+
+                scriptIndex[0] = 6;
+                query = QueryBuilders.rangeQuery("duration").from("5800").toQuery(queryShardContext);
+                topDocs = searcher.search(query, 10);
+                assertEquals(6, topDocs.totalHits.value);
+
+                scriptIndex[0] = 4;
+
+                // Prefix Query
+                query = QueryBuilders.prefixQuery("method", "DE").toQuery(queryShardContext);
+                topDocs = searcher.search(query, 10);
+                assertEquals(2, topDocs.totalHits.value);
+
+                scriptIndex[0] = 4;
+                query = QueryBuilders.wildcardQuery("method", "G*").toQuery(queryShardContext);
+                topDocs = searcher.search(query, 10);
+                assertEquals(7, topDocs.totalHits.value);
+
+                // Regexp Query
+                scriptIndex[0] = 4;
+                query = QueryBuilders.regexpQuery("method", ".*LET.*").toQuery(queryShardContext);
+                topDocs = searcher.search(query, 10);
+                assertEquals(2, topDocs.totalHits.value);
+
+                // GeoPoint Query
+                scriptIndex[0] = 7;
+
+                query = geoShapeQuery("geopoint", new Rectangle(0.0, 55.0, 55.0, 0.0)).toQuery(queryShardContext);
+                topDocs = searcher.search(query, 10);
+                assertEquals(4, topDocs.totalHits.value);
+            }
+        }
+    }
+}
diff --git a/server/src/test/java/org/opensearch/index/mapper/DerivedFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/DerivedFieldMapperTests.java
new file mode 100644
index 0000000000000..e3e2b978369ff
--- /dev/null
+++ b/server/src/test/java/org/opensearch/index/mapper/DerivedFieldMapperTests.java
@@ -0,0 +1,117 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.mapper;
+
+import org.opensearch.common.CheckedConsumer;
+import org.opensearch.common.xcontent.XContentFactory;
+import org.opensearch.core.xcontent.ToXContent;
+import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.script.Script;
+
+import java.io.IOException;
+
+public class DerivedFieldMapperTests extends MapperTestCase {
+
+    protected boolean supportsOrIgnoresBoost() {
+        return false;
+    }
+
+    protected boolean supportsMeta() {
+        return false;
+    }
+
+    // Overriding fieldMapping to make it create derived mappings by default.
+    // This way, the parent tests are checking the right behavior for this Mapper.
+    @Override
+    protected final XContentBuilder fieldMapping(CheckedConsumer<XContentBuilder, IOException> buildField) throws IOException {
+        return fieldMapping(buildField, true);
+    }
+
+    // Overriding this to do nothing since derived fields are not written in the documents
+    @Override
+    protected void writeFieldValue(XContentBuilder builder) throws IOException {}
+
+    @Override
+    protected void minimalMapping(XContentBuilder b) throws IOException {
+        b.field("type", "keyword");
+    }
+
+    protected void registerParameters(ParameterChecker checker) throws IOException {
+        // TODO Any conflicts or updates to check for here? Parameters index, store, doc_values and boost are not
+        // supported for DerivedFieldMapper (we explicitly set these values on initialization)
+    }
+
+    // TODO: Can update this once the query implementation is completed
+    // This is also being left blank because the super assertExistsQuery is trying to parse
+    // an empty source and fails.
+    @Override
+    protected void assertExistsQuery(MapperService mapperService) {}
+
+    public void testSerialization() throws IOException {
+
+        DocumentMapper defaultMapper = createDocumentMapper(fieldMapping(this::minimalMapping));
+        Mapper mapper = defaultMapper.mappers().getMapper("field");
+        assertTrue(mapper instanceof DerivedFieldMapper);
+
+        XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
+        mapper.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        builder.endObject();
+        assertEquals("{\"field\":{\"type\":\"keyword\"}}", builder.toString());
+    }
+
+    public void testParsesScript() throws IOException {
+
+        String scriptString = "doc['test'].value";
+        DocumentMapper defaultMapper = createDocumentMapper(fieldMapping(b -> {
+            b.field("type", "keyword");
+            b.startObject("script");
+            b.field("source", scriptString);
+            b.endObject();
+        }));
+        Mapper mapper = defaultMapper.mappers().getMapper("field");
+        assertTrue(mapper instanceof DerivedFieldMapper);
+        DerivedFieldMapper derivedFieldMapper = (DerivedFieldMapper) mapper;
+        assertEquals(Script.parse(scriptString), derivedFieldMapper.getScript());
+
+        XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
+        mapper.toXContent(builder, ToXContent.EMPTY_PARAMS);
+        builder.endObject();
+        assertEquals(
+            "{\"field\":{\"type\":\"keyword\",\"script\":{\"source\":\"doc['test'].value\",\"lang\":\"painless\"}}}",
+            builder.toString()
+        );
+    }
+
+    // TODO: Is there a case where we want to allow the field to be defined in both 'derived' and 'properties'?
+    // If the user wants to move a field they were testing as derived to be indexed, they should be able to update
+    // the mappings in index template to move the field from 'derived' to 'properties' and it should take affect
+    // during the next index rollover (so even in this case, the field is only defined in one or the other).
+    public void testFieldInDerivedAndProperties() throws IOException {
+        MapperParsingException ex = expectThrows(MapperParsingException.class, () -> createDocumentMapper(topMapping(b -> {
+            b.startObject("derived");
+            b.startObject("field");
+            b.field("type", "keyword");
+            b.endObject();
+            b.endObject();
+            b.startObject("properties");
+            b.startObject("field");
+            b.field("type", "keyword");
+            b.endObject();
+            b.endObject();
+        })));
+        // TODO: Do we want to handle this as a different error? As it stands, it fails as a merge conflict which makes sense.
+        // If it didn't fail here, it would hit the MapperParsingException for the field being defined more than once
+        // when MappingLookup is initialized
+        assertEquals("Failed to parse mapping [_doc]: mapper [field] cannot be changed from type [derived] to [keyword]", ex.getMessage());
+    }
+
+    // TODO TESTCASE: testWithFieldInSource() (derived field with that field present in source)
+    // This is more checking search behavior so may need to revisit this after query implementation
+
+}
diff --git a/server/src/test/java/org/opensearch/index/mapper/DerivedFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/DerivedFieldTypeTests.java
new file mode 100644
index 0000000000000..897848008fd5f
--- /dev/null
+++ b/server/src/test/java/org/opensearch/index/mapper/DerivedFieldTypeTests.java
@@ -0,0 +1,93 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.mapper;
+
+import org.apache.lucene.document.DoubleField;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.InetAddressPoint;
+import org.apache.lucene.document.KeywordField;
+import org.apache.lucene.document.LatLonPoint;
+import org.apache.lucene.document.LongField;
+import org.apache.lucene.document.LongPoint;
+import org.opensearch.common.collect.Tuple;
+import org.opensearch.script.Script;
+
+import java.util.List;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class DerivedFieldTypeTests extends FieldTypeTestCase {
+
+    private DerivedFieldType createDerivedFieldType(String type) {
+        Mapper.BuilderContext context = mock(Mapper.BuilderContext.class);
+        when(context.path()).thenReturn(new ContentPath());
+        return new DerivedFieldType(
+            new DerivedField(type + " _derived_field", type, new Script("")),
+            DerivedFieldSupportedTypes.getFieldMapperFromType(type, type + "_derived_field", context),
+            DerivedFieldSupportedTypes.getIndexableFieldGeneratorType(type, type + "_derived_field")
+        );
+    }
+
+    public void testBooleanType() {
+        DerivedFieldType dft = createDerivedFieldType("boolean");
+        assertTrue(dft.typeFieldMapper instanceof BooleanFieldMapper);
+        assertTrue(dft.indexableFieldGenerator.apply(true) instanceof Field);
+        assertTrue(dft.indexableFieldGenerator.apply(false) instanceof Field);
+    }
+
+    public void testDateType() {
+        DerivedFieldType dft = createDerivedFieldType("date");
+        assertTrue(dft.typeFieldMapper instanceof DateFieldMapper);
+        assertTrue(dft.indexableFieldGenerator.apply(System.currentTimeMillis()) instanceof LongPoint);
+        expectThrows(Exception.class, () -> dft.indexableFieldGenerator.apply("blah"));
+    }
+
+    public void testGeoPointType() {
+        DerivedFieldType dft = createDerivedFieldType("geo_point");
+        assertTrue(dft.typeFieldMapper instanceof GeoPointFieldMapper);
+        assertTrue(dft.indexableFieldGenerator.apply(new Tuple<>(10.0, 20.0)) instanceof LatLonPoint);
+        expectThrows(ClassCastException.class, () -> dft.indexableFieldGenerator.apply(List.of(10.0)));
+        expectThrows(ClassCastException.class, () -> dft.indexableFieldGenerator.apply(List.of()));
+        expectThrows(ClassCastException.class, () -> dft.indexableFieldGenerator.apply(List.of("10")));
+        expectThrows(ClassCastException.class, () -> dft.indexableFieldGenerator.apply(List.of(10.0, 20.0, 30.0)));
+    }
+
+    public void testIPType() {
+        DerivedFieldType dft = createDerivedFieldType("ip");
+        assertTrue(dft.typeFieldMapper instanceof IpFieldMapper);
+        assertTrue(dft.indexableFieldGenerator.apply("127.0.0.1") instanceof InetAddressPoint);
+        expectThrows(Exception.class, () -> dft.indexableFieldGenerator.apply("blah"));
+    }
+
+    public void testKeywordType() {
+        DerivedFieldType dft = createDerivedFieldType("keyword");
+        assertTrue(dft.typeFieldMapper instanceof KeywordFieldMapper);
+        assertTrue(dft.indexableFieldGenerator.apply("test_keyword") instanceof KeywordField);
+        expectThrows(Exception.class, () -> dft.indexableFieldGenerator.apply(10));
+    }
+
+    public void testLongType() {
+        DerivedFieldType dft = createDerivedFieldType("long");
+        assertTrue(dft.typeFieldMapper instanceof NumberFieldMapper);
+        assertTrue(dft.indexableFieldGenerator.apply(10) instanceof LongField);
+        expectThrows(Exception.class, () -> dft.indexableFieldGenerator.apply(10.0));
+    }
+
+    public void testDoubleType() {
+        DerivedFieldType dft = createDerivedFieldType("double");
+        assertTrue(dft.typeFieldMapper instanceof NumberFieldMapper);
+        assertTrue(dft.indexableFieldGenerator.apply(10.0) instanceof DoubleField);
+        expectThrows(Exception.class, () -> dft.indexableFieldGenerator.apply(""));
+    }
+
+    public void testUnsupportedType() {
+        expectThrows(IllegalArgumentException.class, () -> createDerivedFieldType("match_only_text"));
+    }
+}
diff --git a/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldDataTests.java b/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldDataTests.java
index e318ca5e953a3..03ce1eb7252de 100644
--- a/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldDataTests.java
+++ b/server/src/test/java/org/opensearch/index/mapper/FlatObjectFieldDataTests.java
@@ -54,6 +54,49 @@ public void testDocValue() throws Exception {
         assertEquals(1, valueReaders.size());
     }
 
+    public void testLongFieldNameWithHashArray() throws Exception {
+        String mapping = XContentFactory.jsonBuilder()
+            .startObject()
+            .startObject("test")
+            .startObject("properties")
+            .startObject("field")
+            .field("type", FIELD_TYPE)
+            .endObject()
+            .endObject()
+            .endObject()
+            .endObject()
+            .toString();
+        final DocumentMapper mapper = mapperService.documentMapperParser().parse("test", new CompressedXContent(mapping));
+
+        XContentBuilder json = XContentFactory.jsonBuilder()
+            .startObject()
+            .startObject("field")
+            .startObject("detail")
+            .startArray("fooooooooooo")
+            .startObject()
+            .field("name", "baz")
+            .endObject()
+            .startObject()
+            .field("name", "baz")
+            .endObject()
+            .endArray()
+            .endObject()
+            .endObject()
+            .endObject();
+
+        ParsedDocument d = mapper.parse(new SourceToParse("test", "1", BytesReference.bytes(json), MediaTypeRegistry.JSON));
+        writer.addDocument(d.rootDoc());
+        writer.commit();
+
+        IndexFieldData<?> fieldData = getForField("field");
+        List<LeafReaderContext> readers = refreshReader();
+        assertEquals(1, readers.size());
+
+        IndexFieldData<?> valueFieldData = getForField("field._value");
+        List<LeafReaderContext> valueReaders = refreshReader();
+        assertEquals(1, valueReaders.size());
+    }
+
     @Override
     protected String getFieldDataType() {
         return FIELD_TYPE;
diff --git a/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java
index be947935dc4ea..b10a7d8155056 100644
--- a/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java
+++ b/server/src/test/java/org/opensearch/index/mapper/ObjectMapperTests.java
@@ -39,6 +39,7 @@
 import org.opensearch.index.mapper.MapperService.MergeReason;
 import org.opensearch.index.mapper.ObjectMapper.Dynamic;
 import org.opensearch.plugins.Plugin;
+import org.opensearch.script.Script;
 import org.opensearch.test.InternalSettingsPlugin;
 import org.opensearch.test.OpenSearchSingleNodeTestCase;
 
@@ -440,6 +441,52 @@ public void testEmptyName() throws Exception {
         assertThat(e.getMessage(), containsString("name cannot be empty string"));
     }
 
+    public void testDerivedFields() throws Exception {
+        String mapping = XContentFactory.jsonBuilder()
+            .startObject()
+            .startObject("tweet")
+            .startObject("derived")
+            .startObject("derived_field_name1")
+            .field("type", "boolean")
+            .endObject()
+            .startObject("derived_field_name2")
+            .field("type", "keyword")
+            .startObject("script")
+            .field("source", "doc['test'].value")
+            .endObject()
+            .endObject()
+            .endObject()
+            .startObject("properties")
+            .startObject("field_name")
+            .field("type", "date")
+            .endObject()
+            .endObject()
+            .endObject()
+            .endObject()
+            .toString();
+
+        DocumentMapper documentMapper = createIndex("test").mapperService()
+            .documentMapperParser()
+            .parse("tweet", new CompressedXContent(mapping));
+
+        Mapper mapper = documentMapper.root().getMapper("derived_field_name1");
+        assertTrue(mapper instanceof DerivedFieldMapper);
+        DerivedFieldMapper derivedFieldMapper = (DerivedFieldMapper) mapper;
+        assertEquals("boolean", derivedFieldMapper.getType());
+        assertNull(derivedFieldMapper.getScript());
+
+        mapper = documentMapper.root().getMapper("derived_field_name2");
+        assertTrue(mapper instanceof DerivedFieldMapper);
+        derivedFieldMapper = (DerivedFieldMapper) mapper;
+        assertEquals("keyword", derivedFieldMapper.getType());
+        assertEquals(Script.parse("doc['test'].value"), derivedFieldMapper.getScript());
+
+        // Check that field in properties was parsed correctly as well
+        mapper = documentMapper.root().getMapper("field_name");
+        assertNotNull(mapper);
+        assertEquals("date", mapper.typeName());
+    }
+
     @Override
     protected Collection<Class<? extends Plugin>> getPlugins() {
         return pluginList(InternalSettingsPlugin.class);
diff --git a/server/src/test/java/org/opensearch/index/mapper/RangeFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/RangeFieldMapperTests.java
index 331bfb7b2ddf4..91eab942c499a 100644
--- a/server/src/test/java/org/opensearch/index/mapper/RangeFieldMapperTests.java
+++ b/server/src/test/java/org/opensearch/index/mapper/RangeFieldMapperTests.java
@@ -37,6 +37,7 @@
 import org.apache.lucene.index.IndexableField;
 import org.opensearch.common.CheckedConsumer;
 import org.opensearch.common.network.InetAddresses;
+import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.common.xcontent.XContentFactory;
 import org.opensearch.core.xcontent.ToXContent;
 import org.opensearch.core.xcontent.XContentBuilder;
@@ -51,8 +52,10 @@
 import static org.opensearch.index.query.RangeQueryBuilder.GT_FIELD;
 import static org.opensearch.index.query.RangeQueryBuilder.LTE_FIELD;
 import static org.opensearch.index.query.RangeQueryBuilder.LT_FIELD;
+import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.Matchers.anyOf;
 import static org.hamcrest.Matchers.containsString;
+import static org.junit.Assume.assumeThat;
 
 public class RangeFieldMapperTests extends AbstractNumericFieldMapperTestCase {
     private static final String FROM_DATE = "2016-10-31";
@@ -351,7 +354,30 @@ public void testIllegalArguments() throws Exception {
         assertThat(e.getMessage(), containsString("should not define a dateTimeFormatter"));
     }
 
+    public void testSerializeDefaultsLegacy() throws Exception {
+        assumeThat("Using legacy datetime format as default", FeatureFlags.isEnabled(FeatureFlags.DATETIME_FORMATTER_CACHING), is(false));
+
+        for (String type : types()) {
+            DocumentMapper docMapper = createDocumentMapper(fieldMapping(b -> b.field("type", type)));
+            RangeFieldMapper mapper = (RangeFieldMapper) docMapper.root().getMapper("field");
+            XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
+            mapper.doXContentBody(builder, true, ToXContent.EMPTY_PARAMS);
+            String got = builder.endObject().toString();
+
+            // if type is date_range we check that the mapper contains the default format and locale
+            // otherwise it should not contain a locale or format
+            assertTrue(got, got.contains("\"format\":\"strict_date_optional_time||epoch_millis\"") == type.equals("date_range"));
+            assertTrue(got, got.contains("\"locale\":" + "\"" + Locale.ROOT + "\"") == type.equals("date_range"));
+        }
+    }
+
     public void testSerializeDefaults() throws Exception {
+        assumeThat(
+            "Using experimental datetime format as default",
+            FeatureFlags.isEnabled(FeatureFlags.DATETIME_FORMATTER_CACHING),
+            is(true)
+        );
+
         for (String type : types()) {
             DocumentMapper docMapper = createDocumentMapper(fieldMapping(b -> b.field("type", type)));
             RangeFieldMapper mapper = (RangeFieldMapper) docMapper.root().getMapper("field");
diff --git a/server/src/test/java/org/opensearch/index/mapper/RangeFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/RangeFieldTypeTests.java
index 00b48240d0567..49bf227e5073c 100644
--- a/server/src/test/java/org/opensearch/index/mapper/RangeFieldTypeTests.java
+++ b/server/src/test/java/org/opensearch/index/mapper/RangeFieldTypeTests.java
@@ -51,6 +51,7 @@
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.time.DateFormatter;
 import org.opensearch.common.util.BigArrays;
+import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.index.IndexSettings;
 import org.opensearch.index.mapper.DateFieldMapper.DateFieldType;
 import org.opensearch.index.mapper.RangeFieldMapper.RangeFieldType;
@@ -65,8 +66,10 @@
 import java.util.Collections;
 import java.util.Map;
 
+import static org.hamcrest.CoreMatchers.is;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.instanceOf;
+import static org.junit.Assume.assumeThat;
 
 public class RangeFieldTypeTests extends FieldTypeTestCase {
     RangeType type;
@@ -249,7 +252,49 @@ private QueryShardContext createContext() {
         );
     }
 
+    public void testDateRangeQueryUsingMappingFormatLegacy() {
+        assumeThat("Using legacy datetime format as default", FeatureFlags.isEnabled(FeatureFlags.DATETIME_FORMATTER_CACHING), is(false));
+
+        QueryShardContext context = createContext();
+        RangeFieldType strict = new RangeFieldType("field", RangeFieldMapper.Defaults.DATE_FORMATTER);
+        // don't use DISJOINT here because it doesn't work on date fields which we want to compare bounds with
+        ShapeRelation relation = randomValueOtherThan(ShapeRelation.DISJOINT, () -> randomFrom(ShapeRelation.values()));
+
+        // dates will break the default format, month/day of month is turned around in the format
+        final String from = "2016-15-06T15:29:50+08:00";
+        final String to = "2016-16-06T15:29:50+08:00";
+
+        OpenSearchParseException ex = expectThrows(
+            OpenSearchParseException.class,
+            () -> strict.rangeQuery(from, to, true, true, relation, null, null, context)
+        );
+        assertThat(
+            ex.getMessage(),
+            containsString("failed to parse date field [2016-15-06T15:29:50+08:00] with format [strict_date_optional_time||epoch_millis]")
+        );
+
+        // setting mapping format which is compatible with those dates
+        final DateFormatter formatter = DateFormatter.forPattern("yyyy-dd-MM'T'HH:mm:ssZZZZZ");
+        assertEquals(1465975790000L, formatter.parseMillis(from));
+        assertEquals(1466062190000L, formatter.parseMillis(to));
+
+        RangeFieldType fieldType = new RangeFieldType("field", formatter);
+        final Query query = fieldType.rangeQuery(from, to, true, true, relation, null, fieldType.dateMathParser(), context);
+        assertEquals("field:<ranges:[1465975790000 : 1466062190999]>", ((IndexOrDocValuesQuery) query).getIndexQuery().toString());
+
+        // compare lower and upper bounds with what we would get on a `date` field
+        DateFieldType dateFieldType = new DateFieldType("field", DateFieldMapper.Resolution.MILLISECONDS, formatter);
+        final Query queryOnDateField = dateFieldType.rangeQuery(from, to, true, true, relation, null, fieldType.dateMathParser(), context);
+        assertEquals("field:[1465975790000 TO 1466062190999]", ((IndexOrDocValuesQuery) queryOnDateField).getIndexQuery().toString());
+    }
+
     public void testDateRangeQueryUsingMappingFormat() {
+        assumeThat(
+            "Using experimental datetime format as default",
+            FeatureFlags.isEnabled(FeatureFlags.DATETIME_FORMATTER_CACHING),
+            is(true)
+        );
+
         QueryShardContext context = createContext();
         RangeFieldType strict = new RangeFieldType("field", RangeFieldMapper.Defaults.DATE_FORMATTER);
         // don't use DISJOINT here because it doesn't work on date fields which we want to compare bounds with
diff --git a/server/src/test/java/org/opensearch/index/query/DerivedFieldQueryTests.java b/server/src/test/java/org/opensearch/index/query/DerivedFieldQueryTests.java
new file mode 100644
index 0000000000000..5a11ebebb312e
--- /dev/null
+++ b/server/src/test/java/org/opensearch/index/query/DerivedFieldQueryTests.java
@@ -0,0 +1,108 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.query;
+
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.document.KeywordField;
+import org.apache.lucene.document.TextField;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.store.Directory;
+import org.opensearch.common.lucene.Lucene;
+import org.opensearch.index.mapper.DerivedFieldSupportedTypes;
+import org.opensearch.index.mapper.DerivedFieldValueFetcher;
+import org.opensearch.script.DerivedFieldScript;
+import org.opensearch.script.Script;
+import org.opensearch.search.lookup.LeafSearchLookup;
+import org.opensearch.search.lookup.SearchLookup;
+import org.opensearch.search.lookup.SourceLookup;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class DerivedFieldQueryTests extends OpenSearchTestCase {
+
+    private static final String[][] raw_requests = new String[][] {
+        { "40.135.0.0 GET /images/hm_bg.jpg HTTP/1.0", "200", "40.135.0.0" },
+        { "232.0.0.0 GET /images/hm_bg.jpg HTTP/1.0", "400", "232.0.0.0" },
+        { "26.1.0.0 GET /images/hm_bg.jpg HTTP/1.0", "200", "26.1.0.0" },
+        { "247.37.0.0 GET /french/splash_inet.html HTTP/1.0", "400", "247.37.0.0" },
+        { "247.37.0.0 GET /french/splash_inet.html HTTP/1.0", "400", "247.37.0.0" } };
+
+    public void testDerivedField() throws IOException {
+        // Create lucene documents
+        List<Document> docs = new ArrayList<>();
+        for (String[] request : raw_requests) {
+            Document document = new Document();
+            document.add(new TextField("raw_request", request[0], Field.Store.YES));
+            document.add(new KeywordField("status", request[1], Field.Store.YES));
+            docs.add(document);
+        }
+
+        // Mock SearchLookup
+        SearchLookup searchLookup = mock(SearchLookup.class);
+        SourceLookup sourceLookup = new SourceLookup();
+        LeafSearchLookup leafLookup = mock(LeafSearchLookup.class);
+        when(leafLookup.source()).thenReturn(sourceLookup);
+
+        // Mock DerivedFieldScript.Factory
+        DerivedFieldScript.Factory factory = (params, lookup) -> (DerivedFieldScript.LeafFactory) ctx -> {
+            when(searchLookup.getLeafSearchLookup(ctx)).thenReturn(leafLookup);
+            return new DerivedFieldScript(params, lookup, ctx) {
+                @Override
+                public void execute() {
+                    addEmittedValue(raw_requests[sourceLookup.docId()][2]);
+                }
+            };
+        };
+
+        // Create ValueFetcher from mocked DerivedFieldScript.Factory
+        DerivedFieldScript.LeafFactory leafFactory = factory.newFactory((new Script("")).getParams(), searchLookup);
+        DerivedFieldValueFetcher valueFetcher = new DerivedFieldValueFetcher(
+            leafFactory,
+            null,
+            DerivedFieldSupportedTypes.getIndexableFieldGeneratorType("keyword", "ip_from_raw_request")
+        );
+
+        // Create DerivedFieldQuery
+        DerivedFieldQuery derivedFieldQuery = new DerivedFieldQuery(
+            new TermQuery(new Term("ip_from_raw_request", "247.37.0.0")),
+            valueFetcher,
+            searchLookup,
+            Lucene.STANDARD_ANALYZER
+        );
+
+        // Index and Search
+
+        try (Directory dir = newDirectory()) {
+            IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(Lucene.STANDARD_ANALYZER));
+            for (Document d : docs) {
+                iw.addDocument(d);
+            }
+            try (IndexReader reader = DirectoryReader.open(iw)) {
+                iw.close();
+                IndexSearcher searcher = new IndexSearcher(reader);
+                TopDocs topDocs = searcher.search(derivedFieldQuery, 10);
+                assertEquals(2, topDocs.totalHits.value);
+            }
+        }
+    }
+}
diff --git a/server/src/test/java/org/opensearch/index/query/QueryShardContextTests.java b/server/src/test/java/org/opensearch/index/query/QueryShardContextTests.java
index 1a2ad49a3f334..6a7bf10835ddd 100644
--- a/server/src/test/java/org/opensearch/index/query/QueryShardContextTests.java
+++ b/server/src/test/java/org/opensearch/index/query/QueryShardContextTests.java
@@ -31,6 +31,7 @@
 
 package org.opensearch.index.query;
 
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.index.DirectoryReader;
@@ -63,10 +64,17 @@
 import org.opensearch.index.fielddata.LeafFieldData;
 import org.opensearch.index.fielddata.ScriptDocValues;
 import org.opensearch.index.fielddata.plain.AbstractLeafOrdinalsFieldData;
+import org.opensearch.index.mapper.ContentPath;
+import org.opensearch.index.mapper.DerivedField;
+import org.opensearch.index.mapper.DerivedFieldMapper;
+import org.opensearch.index.mapper.DocumentMapper;
 import org.opensearch.index.mapper.IndexFieldMapper;
 import org.opensearch.index.mapper.MappedFieldType;
+import org.opensearch.index.mapper.Mapper;
 import org.opensearch.index.mapper.MapperService;
+import org.opensearch.index.mapper.MappingLookup;
 import org.opensearch.index.mapper.TextFieldMapper;
+import org.opensearch.script.Script;
 import org.opensearch.search.lookup.LeafDocLookup;
 import org.opensearch.search.lookup.LeafSearchLookup;
 import org.opensearch.search.lookup.SearchLookup;
@@ -77,6 +85,7 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.function.BiFunction;
 import java.util.function.Supplier;
 
@@ -118,6 +127,28 @@ public void testFailIfFieldMappingNotFound() {
         assertThat(result.name(), equalTo("name"));
     }
 
+    public void testDerivedFieldMapping() {
+        QueryShardContext context = createQueryShardContext(IndexMetadata.INDEX_UUID_NA_VALUE, null);
+        assertNull(context.failIfFieldMappingNotFound("test_derived", null));
+        context.setDerivedFieldTypes(null);
+        assertNull(context.failIfFieldMappingNotFound("test_derived", null));
+        DocumentMapper documentMapper = mock(DocumentMapper.class);
+        Mapper.BuilderContext builderContext = new Mapper.BuilderContext(Settings.EMPTY, new ContentPath(0));
+        DerivedFieldMapper derivedFieldMapper = new DerivedFieldMapper.Builder(new DerivedField("test_derived", "keyword", new Script("")))
+            .build(builderContext);
+        MappingLookup mappingLookup = new MappingLookup(
+            Collections.singletonList(derivedFieldMapper),
+            Collections.emptyList(),
+            Collections.emptyList(),
+            0,
+            new StandardAnalyzer()
+        );
+        when(documentMapper.mappers()).thenReturn(mappingLookup);
+        context.setDerivedFieldTypes(Map.of("test_derived", derivedFieldMapper.fieldType()));
+        context.setAllowUnmappedFields(false);
+        assertEquals(derivedFieldMapper.fieldType(), context.failIfFieldMappingNotFound("test_derived", null));
+    }
+
     public void testToQueryFails() {
         QueryShardContext context = createQueryShardContext(IndexMetadata.INDEX_UUID_NA_VALUE, null);
         Exception exc = expectThrows(Exception.class, () -> context.toQuery(new AbstractQueryBuilder() {
diff --git a/server/src/test/java/org/opensearch/index/query/TermsQueryBuilderTests.java b/server/src/test/java/org/opensearch/index/query/TermsQueryBuilderTests.java
index 32bf290627b63..97f372dc04a1b 100644
--- a/server/src/test/java/org/opensearch/index/query/TermsQueryBuilderTests.java
+++ b/server/src/test/java/org/opensearch/index/query/TermsQueryBuilderTests.java
@@ -34,11 +34,14 @@
 
 import org.apache.lucene.search.BooleanQuery;
 import org.apache.lucene.search.ConstantScoreQuery;
+import org.apache.lucene.search.FieldExistsQuery;
 import org.apache.lucene.search.IndexOrDocValuesQuery;
+import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.PointInSetQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermInSetQuery;
+import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.util.BytesRef;
 import org.opensearch.OpenSearchException;
 import org.opensearch.action.get.GetRequest;
@@ -137,6 +140,9 @@ protected void doAssertLuceneQuery(TermsQueryBuilder queryBuilder, Query query,
                     .or(instanceOf(ConstantScoreQuery.class))
                     .or(instanceOf(MatchNoDocsQuery.class))
                     .or(instanceOf(IndexOrDocValuesQuery.class))
+                    .or(instanceOf(MatchAllDocsQuery.class))
+                    .or(instanceOf(FieldExistsQuery.class))
+                    .or(instanceOf(TermQuery.class))
             );
             if (query instanceof ConstantScoreQuery) {
                 assertThat(((ConstantScoreQuery) query).getQuery(), instanceOf(BooleanQuery.class));
diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteIndexPathTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteIndexPathTests.java
new file mode 100644
index 0000000000000..8ddbd383756e7
--- /dev/null
+++ b/server/src/test/java/org/opensearch/index/remote/RemoteIndexPathTests.java
@@ -0,0 +1,140 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.remote;
+
+import org.opensearch.common.blobstore.BlobPath;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
+import org.opensearch.core.xcontent.ToXContent;
+import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.core.xcontent.XContentParser;
+import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm;
+import org.opensearch.index.remote.RemoteStoreEnums.PathType;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
+
+import org.mockito.Mockito;
+
+import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.TRANSLOG;
+import static org.opensearch.index.remote.RemoteStoreEnums.DataType.LOCK_FILES;
+
+public class RemoteIndexPathTests extends OpenSearchTestCase {
+
+    /**
+     * This checks that the remote path contains paths only for segment and data/metadata/lock_files combination.
+     */
+    public void testToXContentWithSegmentRepo() throws IOException {
+        RemoteIndexPath indexPath = new RemoteIndexPath(
+            "djjsid73he8yd7usduh",
+            2,
+            new BlobPath().add("djsd878ndjh").add("hcs87cj8"),
+            PathType.HASHED_PREFIX,
+            PathHashAlgorithm.FNV_1A_BASE64,
+            RemoteIndexPath.SEGMENT_PATH
+        );
+        XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(MediaTypeRegistry.JSON);
+        xContentBuilder.startObject();
+        xContentBuilder = indexPath.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS);
+        xContentBuilder.endObject();
+        String expected =
+            "{\"version\":\"1\",\"index_uuid\":\"djjsid73he8yd7usduh\",\"shard_count\":2,\"path_type\":\"HASHED_PREFIX\",\"path_hash_algorithm\":\"FNV_1A_BASE64\",\"path_creation_map\":{\"segments\":[\"data\",\"metadata\",\"lock_files\"]},\"paths\":[\"9BmBinD5HYs/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/segments/data/\",\"ExCNOD8_5ew/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/segments/data/\",\"z8wtf0yr2l4/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/segments/metadata/\",\"VheHVwFlExE/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/segments/metadata/\",\"IgFKbsDeUpQ/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/segments/lock_files/\",\"pA3gy_GZtns/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/segments/lock_files/\"]}";
+        assertEquals(expected, xContentBuilder.toString());
+    }
+
+    /**
+     * This checks that the remote path contains paths only for translog and data/metadata combination.
+     */
+    public void testToXContentForTranslogRepoOnly() throws IOException {
+        RemoteIndexPath indexPath = new RemoteIndexPath(
+            "djjsid73he8yd7usduh",
+            2,
+            new BlobPath().add("djsd878ndjh").add("hcs87cj8"),
+            PathType.HASHED_PREFIX,
+            PathHashAlgorithm.FNV_1A_BASE64,
+            RemoteIndexPath.TRANSLOG_PATH
+        );
+        XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(MediaTypeRegistry.JSON);
+        xContentBuilder.startObject();
+        xContentBuilder = indexPath.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS);
+        xContentBuilder.endObject();
+        String expected =
+            "{\"version\":\"1\",\"index_uuid\":\"djjsid73he8yd7usduh\",\"shard_count\":2,\"path_type\":\"HASHED_PREFIX\",\"path_hash_algorithm\":\"FNV_1A_BASE64\",\"path_creation_map\":{\"translog\":[\"data\",\"metadata\"]},\"paths\":[\"2EaVODaKBck/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/translog/data/\",\"dTS2VqEOUNo/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/translog/data/\",\"PVNKNGonmZw/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/0/translog/metadata/\",\"NXmt0Y6NjA8/djsd878ndjh/hcs87cj8/djjsid73he8yd7usduh/1/translog/metadata/\"]}";
+        assertEquals(expected, xContentBuilder.toString());
+    }
+
+    /**
+     * This checks that the remote path contains paths only for translog and data/metadata combination.
+     */
+    public void testToXContentForBothRepos() throws IOException {
+        Map<RemoteStoreEnums.DataCategory, List<RemoteStoreEnums.DataType>> pathCreationMap = new TreeMap<>();
+        pathCreationMap.putAll(RemoteIndexPath.TRANSLOG_PATH);
+        pathCreationMap.putAll(RemoteIndexPath.SEGMENT_PATH);
+        RemoteIndexPath indexPath = new RemoteIndexPath(
+            "csbdqiu8a7sdnjdks",
+            3,
+            new BlobPath().add("nxf9yv0").add("c3ejoi"),
+            PathType.HASHED_PREFIX,
+            PathHashAlgorithm.FNV_1A_BASE64,
+            pathCreationMap
+        );
+        XContentBuilder xContentBuilder = MediaTypeRegistry.contentBuilder(MediaTypeRegistry.JSON);
+        xContentBuilder.startObject();
+        xContentBuilder = indexPath.toXContent(xContentBuilder, ToXContent.EMPTY_PARAMS);
+        xContentBuilder.endObject();
+        String expected =
+            "{\"version\":\"1\",\"index_uuid\":\"csbdqiu8a7sdnjdks\",\"shard_count\":3,\"path_type\":\"HASHED_PREFIX\",\"path_hash_algorithm\":\"FNV_1A_BASE64\",\"path_creation_map\":{\"translog\":[\"data\",\"metadata\"],\"segments\":[\"data\",\"metadata\",\"lock_files\"]},\"paths\":[\"Cjo0F6kNjYk/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/segments/data/\",\"kpayyhxct1I/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/segments/data/\",\"p2RlgnHeIgc/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/segments/data/\",\"gkPIurBtB1w/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/segments/metadata/\",\"Y4YhlbxAB1c/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/segments/metadata/\",\"HYc8fyVPouI/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/segments/metadata/\",\"igzyZCz1ysI/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/segments/lock_files/\",\"uEluEiYmptk/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/segments/lock_files/\",\"TfAD8f06_7A/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/segments/lock_files/\",\"QqKEpasbEGs/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/translog/data/\",\"sNyoimoe1Bw/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/translog/data/\",\"d4YQtONfq50/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/translog/data/\",\"zLr4UXjK8T4/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/0/translog/metadata/\",\"_s8i7ZmlXGE/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/1/translog/metadata/\",\"tvtD3-k5ISg/nxf9yv0/c3ejoi/csbdqiu8a7sdnjdks/2/translog/metadata/\"]}";
+        assertEquals(expected, xContentBuilder.toString());
+    }
+
+    public void testRemoteIndexPathWithInvalidPathCreationMap() throws IOException {
+        IllegalArgumentException ex = assertThrows(
+            IllegalArgumentException.class,
+            () -> new RemoteIndexPath(
+                "djjsid73he8yd7usduh",
+                2,
+                new BlobPath().add("djsd878ndjh").add("hcs87cj8"),
+                PathType.HASHED_PREFIX,
+                PathHashAlgorithm.FNV_1A_BASE64,
+                new HashMap<>()
+            )
+        );
+        assertEquals(
+            "Invalid input in RemoteIndexPath constructor indexUUID=djjsid73he8yd7usduh shardCount=2 "
+                + "basePath=[djsd878ndjh][hcs87cj8] pathType=HASHED_PREFIX pathHashAlgorithm=FNV_1A_BASE64 pathCreationMap={}",
+            ex.getMessage()
+        );
+    }
+
+    public void testFromXContent() {
+        UnsupportedOperationException ex = assertThrows(
+            UnsupportedOperationException.class,
+            () -> RemoteIndexPath.fromXContent(Mockito.mock(XContentParser.class))
+        );
+        assertEquals("RemoteIndexPath.fromXContent() is not supported", ex.getMessage());
+    }
+
+    public void testInvalidPathCreationMap() {
+        IllegalArgumentException ex = assertThrows(
+            IllegalArgumentException.class,
+            () -> new RemoteIndexPath(
+                "djjsid73he8yd7usduh",
+                2,
+                new BlobPath().add("djsd878ndjh").add("hcs87cj8"),
+                PathType.HASHED_PREFIX,
+                PathHashAlgorithm.FNV_1A_BASE64,
+                Map.of(TRANSLOG, List.of(LOCK_FILES))
+            )
+        );
+        assertEquals("pathCreationMap={TRANSLOG=[LOCK_FILES]} is having illegal combination of category and type", ex.getMessage());
+    }
+}
diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteIndexPathUploaderTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteIndexPathUploaderTests.java
new file mode 100644
index 0000000000000..e539b382a5f3b
--- /dev/null
+++ b/server/src/test/java/org/opensearch/index/remote/RemoteIndexPathUploaderTests.java
@@ -0,0 +1,335 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.remote;
+
+import org.opensearch.Version;
+import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.common.SetOnce;
+import org.opensearch.common.UUIDs;
+import org.opensearch.common.blobstore.AsyncMultiStreamBlobContainer;
+import org.opensearch.common.blobstore.BlobContainer;
+import org.opensearch.common.blobstore.BlobPath;
+import org.opensearch.common.blobstore.BlobStore;
+import org.opensearch.common.compress.DeflateCompressor;
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.TimeValue;
+import org.opensearch.core.action.ActionListener;
+import org.opensearch.gateway.remote.RemoteClusterStateService;
+import org.opensearch.gateway.remote.RemoteClusterStateService.RemoteStateTransferException;
+import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm;
+import org.opensearch.index.remote.RemoteStoreEnums.PathType;
+import org.opensearch.node.Node;
+import org.opensearch.node.remotestore.RemoteStoreNodeAttribute;
+import org.opensearch.repositories.RepositoriesService;
+import org.opensearch.repositories.blobstore.BlobStoreRepository;
+import org.opensearch.test.OpenSearchTestCase;
+import org.opensearch.threadpool.TestThreadPool;
+import org.opensearch.threadpool.ThreadPool;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.mockito.Mockito;
+
+import static org.opensearch.index.remote.RemoteStoreEnums.PathType.FIXED;
+import static org.opensearch.index.remote.RemoteStoreEnums.PathType.HASHED_INFIX;
+import static org.opensearch.index.remote.RemoteStoreEnums.PathType.HASHED_PREFIX;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+public class RemoteIndexPathUploaderTests extends OpenSearchTestCase {
+
+    private static final String CLUSTER_STATE_REPO_KEY = Node.NODE_ATTRIBUTES.getKey()
+        + RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY;
+
+    private static final String TRANSLOG_REPO_NAME = "translog-repo";
+    private static final String SEGMENT_REPO_NAME = "segment-repo";
+
+    private final ThreadPool threadPool = new TestThreadPool(getTestName());
+    private Settings settings;
+    private ClusterSettings clusterSettings;
+    private RepositoriesService repositoriesService;
+    private BlobStoreRepository repository;
+    private BlobStore blobStore;
+    private BlobContainer blobContainer;
+    private BlobPath basePath;
+    private List<IndexMetadata> indexMetadataList;
+    private final AtomicLong successCount = new AtomicLong();
+    private final AtomicLong failureCount = new AtomicLong();
+
+    @Before
+    public void setup() {
+        settings = Settings.builder()
+            .put(RemoteIndexPathUploader.TRANSLOG_REPO_NAME_KEY, TRANSLOG_REPO_NAME)
+            .put(RemoteIndexPathUploader.SEGMENT_REPO_NAME_KEY, TRANSLOG_REPO_NAME)
+            .put(CLUSTER_STATE_REPO_KEY, TRANSLOG_REPO_NAME)
+            .put(RemoteClusterStateService.REMOTE_CLUSTER_STATE_ENABLED_SETTING.getKey(), true)
+            .build();
+        clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        basePath = BlobPath.cleanPath().add("test");
+        repositoriesService = mock(RepositoriesService.class);
+        repository = mock(BlobStoreRepository.class);
+        when(repositoriesService.repository(anyString())).thenReturn(repository);
+        blobStore = mock(BlobStore.class);
+        when(repository.blobStore()).thenReturn(blobStore);
+        when(repositoriesService.repository(TRANSLOG_REPO_NAME)).thenReturn(repository);
+        when(repository.basePath()).thenReturn(basePath);
+        when(repository.getCompressor()).thenReturn(new DeflateCompressor());
+        blobContainer = mock(BlobContainer.class);
+        when(blobStore.blobContainer(any(BlobPath.class))).thenReturn(blobContainer);
+
+        Map<String, String> remoteCustomData = Map.of(
+            PathType.NAME,
+            HASHED_PREFIX.name(),
+            PathHashAlgorithm.NAME,
+            PathHashAlgorithm.FNV_1A_BASE64.name()
+        );
+        Settings idxSettings = Settings.builder()
+            .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)
+            .put(IndexMetadata.SETTING_INDEX_UUID, UUIDs.randomBase64UUID())
+            .build();
+        IndexMetadata indexMetadata = new IndexMetadata.Builder("test").settings(idxSettings)
+            .numberOfShards(1)
+            .numberOfReplicas(0)
+            .putCustom(IndexMetadata.REMOTE_STORE_CUSTOM_KEY, remoteCustomData)
+            .build();
+        indexMetadataList = List.of(indexMetadata);
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        super.tearDown();
+        terminate(threadPool);
+    }
+
+    public void testInterceptWithNoRemoteDataAttributes() {
+        Settings settings = Settings.Builder.EMPTY_SETTINGS;
+        clusterSettings.applySettings(settings);
+        RemoteIndexPathUploader remoteIndexPathUploader = new RemoteIndexPathUploader(
+            threadPool,
+            settings,
+            () -> repositoriesService,
+            clusterSettings
+        );
+        List<IndexMetadata> indexMetadataList = Mockito.<List>mock(List.class);
+        ActionListener<Void> actionListener = ActionListener.wrap(
+            res -> successCount.incrementAndGet(),
+            ex -> failureCount.incrementAndGet()
+        );
+        remoteIndexPathUploader.doOnUpload(indexMetadataList, Collections.emptyMap(), actionListener);
+        assertEquals(1, successCount.get());
+        assertEquals(0, failureCount.get());
+        verify(indexMetadataList, times(0)).stream();
+    }
+
+    public void testInterceptWithEmptyIndexMetadataList() {
+        RemoteIndexPathUploader remoteIndexPathUploader = new RemoteIndexPathUploader(
+            threadPool,
+            settings,
+            () -> repositoriesService,
+            clusterSettings
+        );
+        remoteIndexPathUploader.start();
+        ActionListener<Void> actionListener = ActionListener.wrap(
+            res -> successCount.incrementAndGet(),
+            ex -> failureCount.incrementAndGet()
+        );
+        remoteIndexPathUploader.doOnUpload(Collections.emptyList(), Collections.emptyMap(), actionListener);
+        assertEquals(1, successCount.get());
+        assertEquals(0, failureCount.get());
+    }
+
+    public void testInterceptWithEmptyEligibleIndexMetadataList() {
+        RemoteIndexPathUploader remoteIndexPathUploader = new RemoteIndexPathUploader(
+            threadPool,
+            settings,
+            () -> repositoriesService,
+            clusterSettings
+        );
+        remoteIndexPathUploader.start();
+        ActionListener<Void> actionListener = ActionListener.wrap(
+            res -> successCount.incrementAndGet(),
+            ex -> failureCount.incrementAndGet()
+        );
+
+        // Case 1 - Null remoteCustomData
+        List<IndexMetadata> indexMetadataList = List.of(createIndexMetadata(null));
+        remoteIndexPathUploader.doOnUpload(indexMetadataList, Collections.emptyMap(), actionListener);
+        assertEquals(1, successCount.get());
+        assertEquals(0, failureCount.get());
+
+        // Case 2 - Empty remoteCustomData
+        assertThrows(
+            AssertionError.class,
+            () -> remoteIndexPathUploader.doOnUpload(List.of(createIndexMetadata(new HashMap<>())), Collections.emptyMap(), actionListener)
+        );
+        assertEquals(1, successCount.get());
+        assertEquals(0, failureCount.get());
+
+        // Case 3 - RemoteStoreEnums.PathType.NAME not in remoteCustomData map
+        assertThrows(
+            AssertionError.class,
+            () -> remoteIndexPathUploader.doOnUpload(
+                List.of(createIndexMetadata(Map.of("test", "test"))),
+                Collections.emptyMap(),
+                actionListener
+            )
+        );
+        assertEquals(1, successCount.get());
+        assertEquals(0, failureCount.get());
+
+        // Case 4 - RemoteStoreEnums.PathType.NAME is not HASHED_PREFIX
+        String pathType = randomFrom(FIXED, HASHED_INFIX).name();
+        String pathHashAlgorithm = FIXED.name().equals(pathType) ? null : randomFrom(PathHashAlgorithm.values()).name();
+        Map<String, String> remoteCustomData = new HashMap<>();
+        remoteCustomData.put(PathType.NAME, pathType);
+        remoteCustomData.put(PathHashAlgorithm.NAME, pathHashAlgorithm);
+        indexMetadataList = List.of(createIndexMetadata(remoteCustomData));
+        remoteIndexPathUploader.doOnUpload(indexMetadataList, Collections.emptyMap(), actionListener);
+        assertEquals(2, successCount.get());
+        assertEquals(0, failureCount.get());
+    }
+
+    private IndexMetadata createIndexMetadata(Map<String, String> remoteCustomData) {
+        IndexMetadata.Builder builder = IndexMetadata.builder("test")
+            .settings(Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT))
+            .numberOfShards(1)
+            .numberOfReplicas(0);
+        if (Objects.nonNull(remoteCustomData)) {
+            builder.putCustom(IndexMetadata.REMOTE_STORE_CUSTOM_KEY, remoteCustomData);
+        }
+        return builder.build();
+    }
+
+    public void testInterceptWithSameRepo() throws IOException {
+        RemoteIndexPathUploader remoteIndexPathUploader = new RemoteIndexPathUploader(
+            threadPool,
+            settings,
+            () -> repositoriesService,
+            clusterSettings
+        );
+        remoteIndexPathUploader.start();
+        ActionListener<Void> actionListener = ActionListener.wrap(
+            res -> successCount.incrementAndGet(),
+            ex -> failureCount.incrementAndGet()
+        );
+        remoteIndexPathUploader.doOnUpload(indexMetadataList, Collections.emptyMap(), actionListener);
+        assertEquals(1, successCount.get());
+        assertEquals(0, failureCount.get());
+        verify(blobContainer, times(1)).writeBlob(anyString(), any(InputStream.class), anyLong(), anyBoolean());
+    }
+
+    public void testInterceptWithDifferentRepo() throws IOException {
+        Settings settings = Settings.builder()
+            .put(this.settings)
+            .put(RemoteIndexPathUploader.SEGMENT_REPO_NAME_KEY, SEGMENT_REPO_NAME)
+            .build();
+        when(repositoriesService.repository(SEGMENT_REPO_NAME)).thenReturn(repository);
+        RemoteIndexPathUploader remoteIndexPathUploader = new RemoteIndexPathUploader(
+            threadPool,
+            settings,
+            () -> repositoriesService,
+            clusterSettings
+        );
+        remoteIndexPathUploader.start();
+        ActionListener<Void> actionListener = ActionListener.wrap(
+            res -> successCount.incrementAndGet(),
+            ex -> failureCount.incrementAndGet()
+        );
+        remoteIndexPathUploader.doOnUpload(indexMetadataList, Collections.emptyMap(), actionListener);
+        assertEquals(1, successCount.get());
+        assertEquals(0, failureCount.get());
+        verify(blobContainer, times(2)).writeBlob(anyString(), any(InputStream.class), anyLong(), anyBoolean());
+    }
+
+    public void testInterceptWithLatchAwaitTimeout() throws IOException {
+        blobContainer = mock(AsyncMultiStreamBlobContainer.class);
+        when(blobStore.blobContainer(any(BlobPath.class))).thenReturn(blobContainer);
+        RemoteIndexPathUploader remoteIndexPathUploader = new RemoteIndexPathUploader(
+            threadPool,
+            settings,
+            () -> repositoriesService,
+            clusterSettings
+        );
+        remoteIndexPathUploader.start();
+
+        Settings settings = Settings.builder()
+            .put(this.settings)
+            .put(RemoteClusterStateService.INDEX_METADATA_UPLOAD_TIMEOUT_SETTING.getKey(), TimeValue.ZERO)
+            .build();
+        clusterSettings.applySettings(settings);
+        SetOnce<Exception> exceptionSetOnce = new SetOnce<>();
+        ActionListener<Void> actionListener = ActionListener.wrap(res -> successCount.incrementAndGet(), ex -> {
+            failureCount.incrementAndGet();
+            exceptionSetOnce.set(ex);
+        });
+        remoteIndexPathUploader.doOnUpload(indexMetadataList, Collections.emptyMap(), actionListener);
+        assertEquals(0, successCount.get());
+        assertEquals(1, failureCount.get());
+        assertTrue(exceptionSetOnce.get() instanceof RemoteStateTransferException);
+        assertTrue(
+            exceptionSetOnce.get().getMessage().contains("Timed out waiting while uploading remote index path file for indexes=[test/")
+        );
+        verify(blobContainer, times(0)).writeBlob(anyString(), any(InputStream.class), anyLong(), anyBoolean());
+    }
+
+    public void testInterceptWithInterruptedExceptionDuringLatchAwait() throws Exception {
+        AsyncMultiStreamBlobContainer asyncMultiStreamBlobContainer = mock(AsyncMultiStreamBlobContainer.class);
+        when(blobStore.blobContainer(any(BlobPath.class))).thenReturn(asyncMultiStreamBlobContainer);
+        RemoteIndexPathUploader remoteIndexPathUploader = new RemoteIndexPathUploader(
+            threadPool,
+            settings,
+            () -> repositoriesService,
+            clusterSettings
+        );
+        remoteIndexPathUploader.start();
+        Settings settings = Settings.builder()
+            .put(this.settings)
+            .put(RemoteClusterStateService.INDEX_METADATA_UPLOAD_TIMEOUT_SETTING.getKey(), TimeValue.timeValueSeconds(1))
+            .build();
+        clusterSettings.applySettings(settings);
+        SetOnce<Exception> exceptionSetOnce = new SetOnce<>();
+        ActionListener<Void> actionListener = ActionListener.wrap(res -> successCount.incrementAndGet(), ex -> {
+            failureCount.incrementAndGet();
+            exceptionSetOnce.set(ex);
+        });
+        Thread thread = new Thread(() -> {
+            try {
+                remoteIndexPathUploader.onUpload(indexMetadataList, Collections.emptyMap(), actionListener);
+            } catch (Exception e) {
+                assertTrue(e instanceof InterruptedException);
+                assertEquals("sleep interrupted", e.getMessage());
+            }
+        });
+        thread.start();
+        Thread.sleep(10);
+        thread.interrupt();
+
+        assertBusy(() -> {
+            assertEquals(0, successCount.get());
+            assertEquals(1, failureCount.get());
+        });
+    }
+
+}
diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteMigrationIndexMetadataUpdaterTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteMigrationIndexMetadataUpdaterTests.java
new file mode 100644
index 0000000000000..d8220c93e4eeb
--- /dev/null
+++ b/server/src/test/java/org/opensearch/index/remote/RemoteMigrationIndexMetadataUpdaterTests.java
@@ -0,0 +1,339 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.remote;
+
+import org.opensearch.Version;
+import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.cluster.metadata.Metadata;
+import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.cluster.node.DiscoveryNodes;
+import org.opensearch.cluster.routing.IndexRoutingTable;
+import org.opensearch.cluster.routing.IndexShardRoutingTable;
+import org.opensearch.cluster.routing.RecoverySource;
+import org.opensearch.cluster.routing.RoutingTable;
+import org.opensearch.cluster.routing.ShardRouting;
+import org.opensearch.cluster.routing.ShardRoutingState;
+import org.opensearch.cluster.routing.TestShardRouting;
+import org.opensearch.cluster.routing.UnassignedInfo;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.core.index.Index;
+import org.opensearch.core.index.shard.ShardId;
+import org.opensearch.index.shard.IndexShardTestUtils;
+import org.opensearch.indices.replication.common.ReplicationType;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.UUID;
+
+import static org.opensearch.cluster.metadata.IndexMetadata.REMOTE_STORE_CUSTOM_KEY;
+import static org.opensearch.indices.RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING;
+import static org.opensearch.indices.RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING;
+import static org.mockito.Mockito.mock;
+
+public class RemoteMigrationIndexMetadataUpdaterTests extends OpenSearchTestCase {
+    private final String indexName = "test-index";
+
+    public void testMaybeAddRemoteIndexSettingsAllPrimariesAndReplicasOnRemote() throws IOException {
+        Metadata metadata = createIndexMetadataWithDocrepSettings(indexName);
+        IndexMetadata existingIndexMetadata = metadata.index(indexName);
+        IndexMetadata.Builder indexMetadataBuilder = IndexMetadata.builder(existingIndexMetadata);
+        long currentSettingsVersion = indexMetadataBuilder.settingsVersion();
+        DiscoveryNode primaryNode = IndexShardTestUtils.getFakeRemoteEnabledNode("1");
+        DiscoveryNode replicaNode = IndexShardTestUtils.getFakeRemoteEnabledNode("2");
+        DiscoveryNodes allNodes = DiscoveryNodes.builder().add(primaryNode).add(replicaNode).build();
+        RoutingTable routingTable = createRoutingTableAllShardsStarted(indexName, 1, 1, primaryNode, replicaNode);
+        RemoteMigrationIndexMetadataUpdater migrationIndexMetadataUpdater = new RemoteMigrationIndexMetadataUpdater(
+            allNodes,
+            routingTable,
+            existingIndexMetadata,
+            metadata.settings(),
+            logger
+        );
+        migrationIndexMetadataUpdater.maybeAddRemoteIndexSettings(indexMetadataBuilder, indexName);
+        assertTrue(currentSettingsVersion < indexMetadataBuilder.settingsVersion());
+        assertRemoteSettingsApplied(indexMetadataBuilder.build());
+    }
+
+    public void testMaybeAddRemoteIndexSettingsDoesNotRunWhenSettingsAlreadyPresent() throws IOException {
+        Metadata metadata = createIndexMetadataWithRemoteStoreSettings(indexName);
+        IndexMetadata existingIndexMetadata = metadata.index(indexName);
+        IndexMetadata.Builder indexMetadataBuilder = IndexMetadata.builder(existingIndexMetadata);
+        long currentSettingsVersion = indexMetadataBuilder.settingsVersion();
+        DiscoveryNode primaryNode = IndexShardTestUtils.getFakeRemoteEnabledNode("1");
+        DiscoveryNode replicaNode = IndexShardTestUtils.getFakeRemoteEnabledNode("2");
+        DiscoveryNodes allNodes = DiscoveryNodes.builder().add(primaryNode).add(replicaNode).build();
+        RoutingTable routingTable = createRoutingTableAllShardsStarted(indexName, 1, 1, primaryNode, replicaNode);
+        RemoteMigrationIndexMetadataUpdater migrationIndexMetadataUpdater = new RemoteMigrationIndexMetadataUpdater(
+            allNodes,
+            routingTable,
+            existingIndexMetadata,
+            metadata.settings(),
+            logger
+        );
+        migrationIndexMetadataUpdater.maybeAddRemoteIndexSettings(indexMetadataBuilder, indexName);
+        assertEquals(currentSettingsVersion, indexMetadataBuilder.settingsVersion());
+    }
+
+    public void testMaybeAddRemoteIndexSettingsDoesNotUpdateSettingsWhenAllShardsInDocrep() throws IOException {
+        Metadata metadata = createIndexMetadataWithDocrepSettings(indexName);
+        IndexMetadata existingIndexMetadata = metadata.index(indexName);
+        IndexMetadata.Builder indexMetadataBuilder = IndexMetadata.builder(existingIndexMetadata);
+        long currentSettingsVersion = indexMetadataBuilder.settingsVersion();
+        DiscoveryNode primaryNode = IndexShardTestUtils.getFakeDiscoNode("1");
+        DiscoveryNode replicaNode = IndexShardTestUtils.getFakeDiscoNode("2");
+        DiscoveryNodes allNodes = DiscoveryNodes.builder().add(primaryNode).add(replicaNode).build();
+        RoutingTable routingTable = createRoutingTableAllShardsStarted(indexName, 1, 1, primaryNode, replicaNode);
+        RemoteMigrationIndexMetadataUpdater migrationIndexMetadataUpdater = new RemoteMigrationIndexMetadataUpdater(
+            allNodes,
+            routingTable,
+            existingIndexMetadata,
+            metadata.settings(),
+            logger
+        );
+        migrationIndexMetadataUpdater.maybeAddRemoteIndexSettings(indexMetadataBuilder, indexName);
+        assertEquals(currentSettingsVersion, indexMetadataBuilder.settingsVersion());
+        assertDocrepSettingsApplied(indexMetadataBuilder.build());
+    }
+
+    public void testMaybeAddRemoteIndexSettingsUpdatesIndexSettingsWithUnassignedReplicas() throws IOException {
+        Metadata metadata = createIndexMetadataWithDocrepSettings(indexName);
+        IndexMetadata existingIndexMetadata = metadata.index(indexName);
+        IndexMetadata.Builder indexMetadataBuilder = IndexMetadata.builder(existingIndexMetadata);
+        long currentSettingsVersion = indexMetadataBuilder.settingsVersion();
+        DiscoveryNode primaryNode = IndexShardTestUtils.getFakeRemoteEnabledNode("1");
+        DiscoveryNode replicaNode = IndexShardTestUtils.getFakeDiscoNode("2");
+        DiscoveryNodes allNodes = DiscoveryNodes.builder().add(primaryNode).add(replicaNode).build();
+        RoutingTable routingTable = createRoutingTableReplicasUnassigned(indexName, 1, 1, primaryNode);
+        RemoteMigrationIndexMetadataUpdater migrationIndexMetadataUpdater = new RemoteMigrationIndexMetadataUpdater(
+            allNodes,
+            routingTable,
+            existingIndexMetadata,
+            metadata.settings(),
+            logger
+        );
+        migrationIndexMetadataUpdater.maybeAddRemoteIndexSettings(indexMetadataBuilder, indexName);
+        assertTrue(currentSettingsVersion < indexMetadataBuilder.settingsVersion());
+        assertRemoteSettingsApplied(indexMetadataBuilder.build());
+    }
+
+    public void testMaybeAddRemoteIndexSettingsDoesNotUpdateIndexSettingsWithRelocatingReplicas() throws IOException {
+        Metadata metadata = createIndexMetadataWithDocrepSettings(indexName);
+        IndexMetadata existingIndexMetadata = metadata.index(indexName);
+        IndexMetadata.Builder indexMetadataBuilder = IndexMetadata.builder(existingIndexMetadata);
+        long currentSettingsVersion = indexMetadataBuilder.settingsVersion();
+        DiscoveryNode primaryNode = IndexShardTestUtils.getFakeRemoteEnabledNode("1");
+        DiscoveryNode replicaNode = IndexShardTestUtils.getFakeDiscoNode("2");
+        DiscoveryNode replicaRelocatingNode = IndexShardTestUtils.getFakeDiscoNode("3");
+        DiscoveryNodes allNodes = DiscoveryNodes.builder().add(primaryNode).add(replicaNode).build();
+        RoutingTable routingTable = createRoutingTableReplicasRelocating(indexName, 1, 1, primaryNode, replicaNode, replicaRelocatingNode);
+        RemoteMigrationIndexMetadataUpdater migrationIndexMetadataUpdater = new RemoteMigrationIndexMetadataUpdater(
+            allNodes,
+            routingTable,
+            existingIndexMetadata,
+            metadata.settings(),
+            logger
+        );
+        migrationIndexMetadataUpdater.maybeAddRemoteIndexSettings(indexMetadataBuilder, indexName);
+        assertEquals(currentSettingsVersion, indexMetadataBuilder.settingsVersion());
+        assertDocrepSettingsApplied(indexMetadataBuilder.build());
+    }
+
+    public void testMaybeUpdateRemoteStorePathStrategyExecutes() {
+        Metadata currentMetadata = createIndexMetadataWithDocrepSettings(indexName);
+        IndexMetadata existingIndexMetadata = currentMetadata.index(indexName);
+        IndexMetadata.Builder builder = IndexMetadata.builder(existingIndexMetadata);
+        DiscoveryNodes discoveryNodes = DiscoveryNodes.builder().add(IndexShardTestUtils.getFakeRemoteEnabledNode("1")).build();
+        RemoteMigrationIndexMetadataUpdater migrationIndexMetadataUpdater = new RemoteMigrationIndexMetadataUpdater(
+            discoveryNodes,
+            mock(RoutingTable.class),
+            existingIndexMetadata,
+            Settings.builder()
+                .put(
+                    CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING.getKey(),
+                    RemoteStoreEnums.PathHashAlgorithm.FNV_1A_COMPOSITE_1.name()
+                )
+                .put(CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), RemoteStoreEnums.PathType.HASHED_PREFIX.name())
+                .build(),
+            logger
+        );
+        migrationIndexMetadataUpdater.maybeUpdateRemoteStorePathStrategy(builder, indexName);
+        assertCustomPathMetadataIsPresent(builder.build());
+    }
+
+    public void testMaybeUpdateRemoteStorePathStrategyDoesNotExecute() {
+        Metadata currentMetadata = createIndexMetadataWithRemoteStoreSettings(indexName);
+        IndexMetadata existingIndexMetadata = currentMetadata.index(indexName);
+        IndexMetadata.Builder builder = IndexMetadata.builder(currentMetadata.index(indexName));
+        DiscoveryNodes discoveryNodes = DiscoveryNodes.builder().add(IndexShardTestUtils.getFakeRemoteEnabledNode("1")).build();
+        RemoteMigrationIndexMetadataUpdater migrationIndexMetadataUpdater = new RemoteMigrationIndexMetadataUpdater(
+            discoveryNodes,
+            mock(RoutingTable.class),
+            existingIndexMetadata,
+            Settings.builder()
+                .put(
+                    CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING.getKey(),
+                    RemoteStoreEnums.PathHashAlgorithm.FNV_1A_COMPOSITE_1.name()
+                )
+                .put(CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), RemoteStoreEnums.PathType.HASHED_PREFIX.name())
+                .build(),
+            logger
+        );
+
+        migrationIndexMetadataUpdater.maybeUpdateRemoteStorePathStrategy(builder, indexName);
+
+        assertCustomPathMetadataIsPresent(builder.build());
+    }
+
+    private RoutingTable createRoutingTableAllShardsStarted(
+        String indexName,
+        int numberOfShards,
+        int numberOfReplicas,
+        DiscoveryNode primaryHostingNode,
+        DiscoveryNode replicaHostingNode
+    ) {
+        RoutingTable.Builder builder = RoutingTable.builder();
+        Index index = new Index(indexName, UUID.randomUUID().toString());
+
+        IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(index);
+        for (int i = 0; i < numberOfShards; i++) {
+            ShardId shardId = new ShardId(index, i);
+            IndexShardRoutingTable.Builder indexShardRoutingTable = new IndexShardRoutingTable.Builder(shardId);
+            indexShardRoutingTable.addShard(
+                TestShardRouting.newShardRouting(shardId, primaryHostingNode.getId(), true, ShardRoutingState.STARTED)
+            );
+            for (int j = 0; j < numberOfReplicas; j++) {
+                indexShardRoutingTable.addShard(
+                    TestShardRouting.newShardRouting(shardId, replicaHostingNode.getId(), false, ShardRoutingState.STARTED)
+                );
+            }
+            indexRoutingTableBuilder.addIndexShard(indexShardRoutingTable.build());
+        }
+        return builder.add(indexRoutingTableBuilder.build()).build();
+    }
+
+    private RoutingTable createRoutingTableReplicasUnassigned(
+        String indexName,
+        int numberOfShards,
+        int numberOfReplicas,
+        DiscoveryNode primaryHostingNode
+    ) {
+        RoutingTable.Builder builder = RoutingTable.builder();
+        Index index = new Index(indexName, UUID.randomUUID().toString());
+
+        IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(index);
+        for (int i = 0; i < numberOfShards; i++) {
+            ShardId shardId = new ShardId(index, i);
+            IndexShardRoutingTable.Builder indexShardRoutingTable = new IndexShardRoutingTable.Builder(shardId);
+            indexShardRoutingTable.addShard(
+                TestShardRouting.newShardRouting(shardId, primaryHostingNode.getId(), true, ShardRoutingState.STARTED)
+            );
+            for (int j = 0; j < numberOfReplicas; j++) {
+                indexShardRoutingTable.addShard(
+                    ShardRouting.newUnassigned(
+                        shardId,
+                        false,
+                        RecoverySource.PeerRecoverySource.INSTANCE,
+                        new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, null)
+                    )
+                );
+            }
+            indexRoutingTableBuilder.addIndexShard(indexShardRoutingTable.build());
+        }
+        return builder.add(indexRoutingTableBuilder.build()).build();
+    }
+
+    private RoutingTable createRoutingTableReplicasRelocating(
+        String indexName,
+        int numberOfShards,
+        int numberOfReplicas,
+        DiscoveryNode primaryHostingNodes,
+        DiscoveryNode replicaHostingNode,
+        DiscoveryNode replicaRelocatingNode
+    ) {
+        RoutingTable.Builder builder = RoutingTable.builder();
+        Index index = new Index(indexName, UUID.randomUUID().toString());
+
+        IndexRoutingTable.Builder indexRoutingTableBuilder = IndexRoutingTable.builder(index);
+        for (int i = 0; i < numberOfShards; i++) {
+            ShardId shardId = new ShardId(index, i);
+            IndexShardRoutingTable.Builder indexShardRoutingTable = new IndexShardRoutingTable.Builder(shardId);
+            indexShardRoutingTable.addShard(
+                TestShardRouting.newShardRouting(shardId, primaryHostingNodes.getId(), true, ShardRoutingState.STARTED)
+            );
+            for (int j = 0; j < numberOfReplicas; j++) {
+                indexShardRoutingTable.addShard(
+                    TestShardRouting.newShardRouting(
+                        shardId,
+                        replicaHostingNode.getId(),
+                        replicaRelocatingNode.getId(),
+                        false,
+                        ShardRoutingState.RELOCATING
+                    )
+                );
+            }
+            indexRoutingTableBuilder.addIndexShard(indexShardRoutingTable.build());
+        }
+        return builder.add(indexRoutingTableBuilder.build()).build();
+    }
+
+    public static Metadata createIndexMetadataWithRemoteStoreSettings(String indexName) {
+        IndexMetadata.Builder indexMetadata = IndexMetadata.builder(indexName);
+        indexMetadata.settings(
+            Settings.builder()
+                .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)
+                .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)
+                .put(IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING.getKey(), true)
+                .put(IndexMetadata.INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING.getKey(), "dummy-tlog-repo")
+                .put(IndexMetadata.INDEX_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING.getKey(), "dummy-segment-repo")
+                .put(IndexMetadata.INDEX_REPLICATION_TYPE_SETTING.getKey(), "SEGMENT")
+                .build()
+        )
+            .putCustom(
+                REMOTE_STORE_CUSTOM_KEY,
+                Map.of(RemoteStoreEnums.PathType.NAME, "dummy", RemoteStoreEnums.PathHashAlgorithm.NAME, "dummy")
+            )
+            .build();
+        return Metadata.builder().put(indexMetadata).build();
+    }
+
+    public static Metadata createIndexMetadataWithDocrepSettings(String indexName) {
+        IndexMetadata.Builder indexMetadata = IndexMetadata.builder(indexName);
+        indexMetadata.settings(
+            Settings.builder()
+                .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+                .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)
+                .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)
+                .put(IndexMetadata.INDEX_REPLICATION_TYPE_SETTING.getKey(), "DOCUMENT")
+                .build()
+        ).build();
+        return Metadata.builder().put(indexMetadata).build();
+    }
+
+    private void assertRemoteSettingsApplied(IndexMetadata indexMetadata) {
+        assertTrue(IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING.get(indexMetadata.getSettings()));
+        assertTrue(IndexMetadata.INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING.exists(indexMetadata.getSettings()));
+        assertTrue(IndexMetadata.INDEX_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING.exists(indexMetadata.getSettings()));
+        assertEquals(ReplicationType.SEGMENT, IndexMetadata.INDEX_REPLICATION_TYPE_SETTING.get(indexMetadata.getSettings()));
+    }
+
+    private void assertDocrepSettingsApplied(IndexMetadata indexMetadata) {
+        assertFalse(IndexMetadata.INDEX_REMOTE_STORE_ENABLED_SETTING.get(indexMetadata.getSettings()));
+        assertFalse(IndexMetadata.INDEX_REMOTE_TRANSLOG_REPOSITORY_SETTING.exists(indexMetadata.getSettings()));
+        assertFalse(IndexMetadata.INDEX_REMOTE_SEGMENT_STORE_REPOSITORY_SETTING.exists(indexMetadata.getSettings()));
+        assertEquals(ReplicationType.DOCUMENT, IndexMetadata.INDEX_REPLICATION_TYPE_SETTING.get(indexMetadata.getSettings()));
+    }
+
+    private void assertCustomPathMetadataIsPresent(IndexMetadata indexMetadata) {
+        assertNotNull(indexMetadata.getCustomData(REMOTE_STORE_CUSTOM_KEY));
+        assertNotNull(indexMetadata.getCustomData(REMOTE_STORE_CUSTOM_KEY).get(RemoteStoreEnums.PathType.NAME));
+        assertNotNull(indexMetadata.getCustomData(REMOTE_STORE_CUSTOM_KEY).get(RemoteStoreEnums.PathHashAlgorithm.NAME));
+    }
+}
diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteStoreEnumsTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteStoreEnumsTests.java
new file mode 100644
index 0000000000000..575b397382f24
--- /dev/null
+++ b/server/src/test/java/org/opensearch/index/remote/RemoteStoreEnumsTests.java
@@ -0,0 +1,711 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.remote;
+
+import org.apache.logging.log4j.message.ParameterizedMessage;
+import org.opensearch.common.blobstore.BlobPath;
+import org.opensearch.index.remote.RemoteStoreEnums.DataCategory;
+import org.opensearch.index.remote.RemoteStoreEnums.DataType;
+import org.opensearch.index.remote.RemoteStoreEnums.PathType;
+import org.opensearch.index.remote.RemoteStorePathStrategy.PathInput;
+import org.opensearch.test.OpenSearchTestCase;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Locale;
+
+import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.SEGMENTS;
+import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.TRANSLOG;
+import static org.opensearch.index.remote.RemoteStoreEnums.DataType.DATA;
+import static org.opensearch.index.remote.RemoteStoreEnums.DataType.LOCK_FILES;
+import static org.opensearch.index.remote.RemoteStoreEnums.DataType.METADATA;
+import static org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm.FNV_1A_BASE64;
+import static org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm.FNV_1A_COMPOSITE_1;
+import static org.opensearch.index.remote.RemoteStoreEnums.PathType.FIXED;
+import static org.opensearch.index.remote.RemoteStoreEnums.PathType.HASHED_INFIX;
+import static org.opensearch.index.remote.RemoteStoreEnums.PathType.HASHED_PREFIX;
+import static org.opensearch.index.remote.RemoteStoreEnums.PathType.parseString;
+
+public class RemoteStoreEnumsTests extends OpenSearchTestCase {
+
+    private static final String SEPARATOR = "/";
+
+    public void testParseString() {
+        // Case 1 - Pass values from the enum.
+        String typeString = FIXED.toString();
+        PathType type = parseString(randomFrom(typeString, typeString.toLowerCase(Locale.ROOT)));
+        assertEquals(FIXED, type);
+
+        typeString = PathType.HASHED_PREFIX.toString();
+        type = parseString(randomFrom(typeString, typeString.toLowerCase(Locale.ROOT)));
+        assertEquals(PathType.HASHED_PREFIX, type);
+
+        // Case 2 - Pass random string
+        String randomTypeString = randomAlphaOfLength(2);
+        IllegalArgumentException ex = assertThrows(IllegalArgumentException.class, () -> parseString(randomTypeString));
+        assertEquals("Could not parse PathType for [" + randomTypeString + "]", ex.getMessage());
+
+        // Case 3 - Null string
+        ex = assertThrows(IllegalArgumentException.class, () -> parseString(null));
+        assertEquals("Could not parse PathType for [null]", ex.getMessage());
+    }
+
+    public void testGeneratePathForFixedType() {
+        BlobPath blobPath = new BlobPath();
+        List<String> pathList = getPathList();
+        for (String path : pathList) {
+            blobPath = blobPath.add(path);
+        }
+
+        String indexUUID = randomAlphaOfLength(10);
+        String shardId = String.valueOf(randomInt(100));
+        DataCategory dataCategory = TRANSLOG;
+        DataType dataType = DATA;
+
+        String basePath = getPath(pathList) + indexUUID + SEPARATOR + shardId + SEPARATOR;
+        // Translog Data
+        PathInput pathInput = PathInput.builder()
+            .basePath(blobPath)
+            .indexUUID(indexUUID)
+            .shardId(shardId)
+            .dataCategory(dataCategory)
+            .dataType(dataType)
+            .build();
+        BlobPath result = FIXED.path(pathInput, null);
+        assertEquals(basePath + dataCategory.getName() + SEPARATOR + dataType.getName() + SEPARATOR, result.buildAsString());
+
+        // Translog Metadata
+        dataType = METADATA;
+        pathInput = PathInput.builder()
+            .basePath(blobPath)
+            .indexUUID(indexUUID)
+            .shardId(shardId)
+            .dataCategory(dataCategory)
+            .dataType(dataType)
+            .build();
+        result = FIXED.path(pathInput, null);
+        assertEquals(basePath + dataCategory.getName() + SEPARATOR + dataType.getName() + SEPARATOR, result.buildAsString());
+
+        // Translog Lock files - This is a negative case where the assertion will trip.
+        dataType = LOCK_FILES;
+        PathInput finalPathInput = PathInput.builder()
+            .basePath(blobPath)
+            .indexUUID(indexUUID)
+            .shardId(shardId)
+            .dataCategory(dataCategory)
+            .dataType(dataType)
+            .build();
+        assertThrows(AssertionError.class, () -> FIXED.path(finalPathInput, null));
+
+        // Segment Data
+        dataCategory = SEGMENTS;
+        dataType = DATA;
+        pathInput = PathInput.builder()
+            .basePath(blobPath)
+            .indexUUID(indexUUID)
+            .shardId(shardId)
+            .dataCategory(dataCategory)
+            .dataType(dataType)
+            .build();
+        result = FIXED.path(pathInput, null);
+        assertEquals(basePath + dataCategory.getName() + SEPARATOR + dataType.getName() + SEPARATOR, result.buildAsString());
+
+        // Segment Metadata
+        dataType = METADATA;
+        pathInput = PathInput.builder()
+            .basePath(blobPath)
+            .indexUUID(indexUUID)
+            .shardId(shardId)
+            .dataCategory(dataCategory)
+            .dataType(dataType)
+            .build();
+        result = FIXED.path(pathInput, null);
+        assertEquals(basePath + dataCategory.getName() + SEPARATOR + dataType.getName() + SEPARATOR, result.buildAsString());
+
+        // Segment Metadata
+        dataType = LOCK_FILES;
+        pathInput = PathInput.builder()
+            .basePath(blobPath)
+            .indexUUID(indexUUID)
+            .shardId(shardId)
+            .dataCategory(dataCategory)
+            .dataType(dataType)
+            .build();
+        result = FIXED.path(pathInput, null);
+        assertEquals(basePath + dataCategory.getName() + SEPARATOR + dataType.getName() + SEPARATOR, result.buildAsString());
+    }
+
+    public void testGeneratePathForHashedPrefixType() {
+        BlobPath blobPath = new BlobPath();
+        List<String> pathList = getPathList();
+        for (String path : pathList) {
+            blobPath = blobPath.add(path);
+        }
+
+        String indexUUID = randomAlphaOfLength(10);
+        String shardId = String.valueOf(randomInt(100));
+        DataCategory dataCategory = TRANSLOG;
+        DataType dataType = DATA;
+
+        String basePath = getPath(pathList) + indexUUID + SEPARATOR + shardId;
+        // Translog Data
+        PathInput pathInput = PathInput.builder()
+            .basePath(blobPath)
+            .indexUUID(indexUUID)
+            .shardId(shardId)
+            .dataCategory(dataCategory)
+            .dataType(dataType)
+            .build();
+        BlobPath result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64);
+        assertTrue(
+            result.buildAsString()
+                .startsWith(String.join(SEPARATOR, FNV_1A_BASE64.hash(pathInput), basePath, dataCategory.getName(), dataType.getName()))
+        );
+
+        // assert with exact value for known base path
+        BlobPath fixedBlobPath = BlobPath.cleanPath().add("xjsdhj").add("ddjsha").add("yudy7sd").add("32hdhua7").add("89jdij");
+        String fixedIndexUUID = "k2ijhe877d7yuhx7";
+        String fixedShardId = "10";
+        pathInput = PathInput.builder()
+            .basePath(fixedBlobPath)
+            .indexUUID(fixedIndexUUID)
+            .shardId(fixedShardId)
+            .dataCategory(dataCategory)
+            .dataType(dataType)
+            .build();
+        result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64);
+        assertEquals("DgSI70IciXs/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/translog/data/", result.buildAsString());
+
+        // Translog Metadata
+        dataType = METADATA;
+        pathInput = PathInput.builder()
+            .basePath(blobPath)
+            .indexUUID(indexUUID)
+            .shardId(shardId)
+            .dataCategory(dataCategory)
+            .dataType(dataType)
+            .build();
+        result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64);
+        assertTrue(
+            result.buildAsString()
+                .startsWith(String.join(SEPARATOR, FNV_1A_BASE64.hash(pathInput), basePath, dataCategory.getName(), dataType.getName()))
+        );
+
+        // assert with exact value for known base path
+        pathInput = PathInput.builder()
+            .basePath(fixedBlobPath)
+            .indexUUID(fixedIndexUUID)
+            .shardId(fixedShardId)
+            .dataCategory(dataCategory)
+            .dataType(dataType)
+            .build();
+        result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64);
+        assertEquals("oKU5SjILiy4/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/translog/metadata/", result.buildAsString());
+
+        // Translog Lock files - This is a negative case where the assertion will trip.
+        dataType = LOCK_FILES;
+        PathInput finalPathInput = PathInput.builder()
+            .basePath(blobPath)
+            .indexUUID(indexUUID)
+            .shardId(shardId)
+            .dataCategory(dataCategory)
+            .dataType(dataType)
+            .build();
+        assertThrows(AssertionError.class, () -> HASHED_PREFIX.path(finalPathInput, null));
+
+        // assert with exact value for known base path
+        pathInput = PathInput.builder()
+            .basePath(fixedBlobPath)
+            .indexUUID(fixedIndexUUID)
+            .shardId(fixedShardId)
+            .dataCategory(dataCategory)
+            .dataType(dataType)
+            .build();
+        assertThrows(AssertionError.class, () -> HASHED_PREFIX.path(finalPathInput, null));
+
+        // Segment Data
+        dataCategory = SEGMENTS;
+        dataType = DATA;
+        pathInput = PathInput.builder()
+            .basePath(blobPath)
+            .indexUUID(indexUUID)
+            .shardId(shardId)
+            .dataCategory(dataCategory)
+            .dataType(dataType)
+            .build();
+        result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64);
+        assertTrue(
+            result.buildAsString()
+                .startsWith(String.join(SEPARATOR, FNV_1A_BASE64.hash(pathInput), basePath, dataCategory.getName(), dataType.getName()))
+        );
+
+        // assert with exact value for known base path
+        pathInput = PathInput.builder()
+            .basePath(fixedBlobPath)
+            .indexUUID(fixedIndexUUID)
+            .shardId(fixedShardId)
+            .dataCategory(dataCategory)
+            .dataType(dataType)
+            .build();
+        result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64);
+        assertEquals("AUBRfCIuWdk/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/data/", result.buildAsString());
+
+        // Segment Metadata
+        dataType = METADATA;
+        pathInput = PathInput.builder()
+            .basePath(blobPath)
+            .indexUUID(indexUUID)
+            .shardId(shardId)
+            .dataCategory(dataCategory)
+            .dataType(dataType)
+            .build();
+        result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64);
+        assertTrue(
+            result.buildAsString()
+                .startsWith(String.join(SEPARATOR, FNV_1A_BASE64.hash(pathInput), basePath, dataCategory.getName(), dataType.getName()))
+        );
+
+        // assert with exact value for known base path
+        pathInput = PathInput.builder()
+            .basePath(fixedBlobPath)
+            .indexUUID(fixedIndexUUID)
+            .shardId(fixedShardId)
+            .dataCategory(dataCategory)
+            .dataType(dataType)
+            .build();
+        result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64);
+        assertEquals("erwR-G735Uw/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/metadata/", result.buildAsString());
+
+        // Segment Lockfiles
+        dataType = LOCK_FILES;
+        pathInput = PathInput.builder()
+            .basePath(blobPath)
+            .indexUUID(indexUUID)
+            .shardId(shardId)
+            .dataCategory(dataCategory)
+            .dataType(dataType)
+            .build();
+        result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64);
+        assertTrue(
+            result.buildAsString()
+                .startsWith(String.join(SEPARATOR, FNV_1A_BASE64.hash(pathInput), basePath, dataCategory.getName(), dataType.getName()))
+        );
+
+        // assert with exact value for known base path
+        pathInput = PathInput.builder()
+            .basePath(fixedBlobPath)
+            .indexUUID(fixedIndexUUID)
+            .shardId(fixedShardId)
+            .dataCategory(dataCategory)
+            .dataType(dataType)
+            .build();
+        result = HASHED_PREFIX.path(pathInput, FNV_1A_BASE64);
+        assertEquals("KeYDIk0mJXI/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/lock_files/", result.buildAsString());
+    }
+
+    public void testGeneratePathForHashedPrefixTypeAndFNVCompositeHashAlgorithm() {
+        BlobPath blobPath = new BlobPath();
+        List<String> pathList = getPathList();
+        for (String path : pathList) {
+            blobPath = blobPath.add(path);
+        }
+
+        String indexUUID = randomAlphaOfLength(10);
+        String shardId = String.valueOf(randomInt(100));
+        DataCategory dataCategory = TRANSLOG;
+        DataType dataType = DATA;
+
+        String basePath = getPath(pathList) + indexUUID + SEPARATOR + shardId;
+        // Translog Data
+        PathInput pathInput = PathInput.builder()
+            .basePath(blobPath)
+            .indexUUID(indexUUID)
+            .shardId(shardId)
+            .dataCategory(dataCategory)
+            .dataType(dataType)
+            .build();
+        BlobPath result = HASHED_PREFIX.path(pathInput, FNV_1A_COMPOSITE_1);
+        assertTrue(
+            result.buildAsString()
+                .startsWith(
+                    String.join(SEPARATOR, FNV_1A_COMPOSITE_1.hash(pathInput), basePath, dataCategory.getName(), dataType.getName())
+                )
+        );
+
+        // assert with exact value for known base path
+        BlobPath fixedBlobPath = BlobPath.cleanPath().add("xjsdhj").add("ddjsha").add("yudy7sd").add("32hdhua7").add("89jdij");
+        String fixedIndexUUID = "k2ijhe877d7yuhx7";
+        String fixedShardId = "10";
+        pathInput = PathInput.builder()
+            .basePath(fixedBlobPath)
+            .indexUUID(fixedIndexUUID)
+            .shardId(fixedShardId)
+            .dataCategory(dataCategory)
+            .dataType(dataType)
+            .build();
+        result = HASHED_PREFIX.path(pathInput, FNV_1A_COMPOSITE_1);
+        assertEquals("D10000001001000/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/translog/data/", result.buildAsString());
+
+        // Translog Metadata
+        dataType = METADATA;
+        pathInput = PathInput.builder()
+            .basePath(blobPath)
+            .indexUUID(indexUUID)
+            .shardId(shardId)
+            .dataCategory(dataCategory)
+            .dataType(dataType)
+            .build();
+        result = HASHED_PREFIX.path(pathInput, FNV_1A_COMPOSITE_1);
+        assertTrue(
+            result.buildAsString()
+                .startsWith(
+                    String.join(SEPARATOR, FNV_1A_COMPOSITE_1.hash(pathInput), basePath, dataCategory.getName(), dataType.getName())
+                )
+        );
+
+        // assert with exact value for known base path
+        pathInput = PathInput.builder()
+            .basePath(fixedBlobPath)
+            .indexUUID(fixedIndexUUID)
+            .shardId(fixedShardId)
+            .dataCategory(dataCategory)
+            .dataType(dataType)
+            .build();
+        result = HASHED_PREFIX.path(pathInput, FNV_1A_COMPOSITE_1);
+        assertEquals(
+            "o00101001010011/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/translog/metadata/",
+            result.buildAsString()
+        );
+
+        // Translog Lock files - This is a negative case where the assertion will trip.
+        dataType = LOCK_FILES;
+        PathInput finalPathInput = PathInput.builder()
+            .basePath(blobPath)
+            .indexUUID(indexUUID)
+            .shardId(shardId)
+            .dataCategory(dataCategory)
+            .dataType(dataType)
+            .build();
+        assertThrows(AssertionError.class, () -> HASHED_PREFIX.path(finalPathInput, null));
+
+        // assert with exact value for known base path
+        pathInput = PathInput.builder()
+            .basePath(fixedBlobPath)
+            .indexUUID(fixedIndexUUID)
+            .shardId(fixedShardId)
+            .dataCategory(dataCategory)
+            .dataType(dataType)
+            .build();
+        assertThrows(AssertionError.class, () -> HASHED_PREFIX.path(finalPathInput, null));
+
+        // Segment Data
+        dataCategory = SEGMENTS;
+        dataType = DATA;
+        pathInput = PathInput.builder()
+            .basePath(blobPath)
+            .indexUUID(indexUUID)
+            .shardId(shardId)
+            .dataCategory(dataCategory)
+            .dataType(dataType)
+            .build();
+        result = HASHED_PREFIX.path(pathInput, FNV_1A_COMPOSITE_1);
+        assertTrue(
+            result.buildAsString()
+                .startsWith(
+                    String.join(SEPARATOR, FNV_1A_COMPOSITE_1.hash(pathInput), basePath, dataCategory.getName(), dataType.getName())
+                )
+        );
+
+        // assert with exact value for known base path
+        pathInput = PathInput.builder()
+            .basePath(fixedBlobPath)
+            .indexUUID(fixedIndexUUID)
+            .shardId(fixedShardId)
+            .dataCategory(dataCategory)
+            .dataType(dataType)
+            .build();
+        result = HASHED_PREFIX.path(pathInput, FNV_1A_COMPOSITE_1);
+        assertEquals("A01010000000101/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/data/", result.buildAsString());
+
+        // Segment Metadata
+        dataType = METADATA;
+        pathInput = PathInput.builder()
+            .basePath(blobPath)
+            .indexUUID(indexUUID)
+            .shardId(shardId)
+            .dataCategory(dataCategory)
+            .dataType(dataType)
+            .build();
+        result = HASHED_PREFIX.path(pathInput, FNV_1A_COMPOSITE_1);
+        assertTrue(
+            result.buildAsString()
+                .startsWith(
+                    String.join(SEPARATOR, FNV_1A_COMPOSITE_1.hash(pathInput), basePath, dataCategory.getName(), dataType.getName())
+                )
+        );
+
+        // assert with exact value for known base path
+        pathInput = PathInput.builder()
+            .basePath(fixedBlobPath)
+            .indexUUID(fixedIndexUUID)
+            .shardId(fixedShardId)
+            .dataCategory(dataCategory)
+            .dataType(dataType)
+            .build();
+        result = HASHED_PREFIX.path(pathInput, FNV_1A_COMPOSITE_1);
+        assertEquals(
+            "e10101111000001/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/metadata/",
+            result.buildAsString()
+        );
+
+        // Segment Lockfiles
+        dataType = LOCK_FILES;
+        pathInput = PathInput.builder()
+            .basePath(blobPath)
+            .indexUUID(indexUUID)
+            .shardId(shardId)
+            .dataCategory(dataCategory)
+            .dataType(dataType)
+            .build();
+        result = HASHED_PREFIX.path(pathInput, FNV_1A_COMPOSITE_1);
+        assertTrue(
+            result.buildAsString()
+                .startsWith(
+                    String.join(SEPARATOR, FNV_1A_COMPOSITE_1.hash(pathInput), basePath, dataCategory.getName(), dataType.getName())
+                )
+        );
+
+        // assert with exact value for known base path
+        pathInput = PathInput.builder()
+            .basePath(fixedBlobPath)
+            .indexUUID(fixedIndexUUID)
+            .shardId(fixedShardId)
+            .dataCategory(dataCategory)
+            .dataType(dataType)
+            .build();
+        result = HASHED_PREFIX.path(pathInput, FNV_1A_COMPOSITE_1);
+        assertEquals(
+            "K01111001100000/xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/k2ijhe877d7yuhx7/10/segments/lock_files/",
+            result.buildAsString()
+        );
+    }
+
+    public void testGeneratePathForHashedInfixType() {
+        BlobPath blobPath = new BlobPath();
+        List<String> pathList = getPathList();
+        for (String path : pathList) {
+            blobPath = blobPath.add(path);
+        }
+
+        String indexUUID = randomAlphaOfLength(10);
+        String shardId = String.valueOf(randomInt(100));
+        DataCategory dataCategory = TRANSLOG;
+        DataType dataType = DATA;
+
+        String basePath = getPath(pathList);
+        basePath = basePath.length() == 0 ? basePath : basePath.substring(0, basePath.length() - 1);
+        // Translog Data
+        PathInput pathInput = PathInput.builder()
+            .basePath(blobPath)
+            .indexUUID(indexUUID)
+            .shardId(shardId)
+            .dataCategory(dataCategory)
+            .dataType(dataType)
+            .build();
+        BlobPath result = HASHED_INFIX.path(pathInput, FNV_1A_BASE64);
+        String expected = derivePath(basePath, pathInput);
+        String actual = result.buildAsString();
+        assertTrue(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual.startsWith(expected));
+
+        // assert with exact value for known base path
+        BlobPath fixedBlobPath = BlobPath.cleanPath().add("xjsdhj").add("ddjsha").add("yudy7sd").add("32hdhua7").add("89jdij");
+        String fixedIndexUUID = "k2ijhe877d7yuhx7";
+        String fixedShardId = "10";
+        pathInput = PathInput.builder()
+            .basePath(fixedBlobPath)
+            .indexUUID(fixedIndexUUID)
+            .shardId(fixedShardId)
+            .dataCategory(dataCategory)
+            .dataType(dataType)
+            .build();
+        result = HASHED_INFIX.path(pathInput, FNV_1A_BASE64);
+        expected = "xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/DgSI70IciXs/k2ijhe877d7yuhx7/10/translog/data/";
+        actual = result.buildAsString();
+        assertTrue(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual.startsWith(expected));
+
+        // Translog Metadata
+        dataType = METADATA;
+        pathInput = PathInput.builder()
+            .basePath(blobPath)
+            .indexUUID(indexUUID)
+            .shardId(shardId)
+            .dataCategory(dataCategory)
+            .dataType(dataType)
+            .build();
+
+        result = HASHED_INFIX.path(pathInput, FNV_1A_BASE64);
+        expected = derivePath(basePath, pathInput);
+        actual = result.buildAsString();
+        assertTrue(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual.startsWith(expected));
+
+        // assert with exact value for known base path
+        pathInput = PathInput.builder()
+            .basePath(fixedBlobPath)
+            .indexUUID(fixedIndexUUID)
+            .shardId(fixedShardId)
+            .dataCategory(dataCategory)
+            .dataType(dataType)
+            .build();
+        result = HASHED_INFIX.path(pathInput, FNV_1A_BASE64);
+        expected = "xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/oKU5SjILiy4/k2ijhe877d7yuhx7/10/translog/metadata/";
+        actual = result.buildAsString();
+        assertTrue(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual.startsWith(expected));
+
+        // Translog Lock files - This is a negative case where the assertion will trip.
+        dataType = LOCK_FILES;
+        PathInput finalPathInput = PathInput.builder()
+            .basePath(blobPath)
+            .indexUUID(indexUUID)
+            .shardId(shardId)
+            .dataCategory(dataCategory)
+            .dataType(dataType)
+            .build();
+        assertThrows(AssertionError.class, () -> HASHED_INFIX.path(finalPathInput, null));
+
+        // assert with exact value for known base path
+        pathInput = PathInput.builder()
+            .basePath(fixedBlobPath)
+            .indexUUID(fixedIndexUUID)
+            .shardId(fixedShardId)
+            .dataCategory(dataCategory)
+            .dataType(dataType)
+            .build();
+        assertThrows(AssertionError.class, () -> HASHED_INFIX.path(finalPathInput, null));
+
+        // Segment Data
+        dataCategory = SEGMENTS;
+        dataType = DATA;
+        pathInput = PathInput.builder()
+            .basePath(blobPath)
+            .indexUUID(indexUUID)
+            .shardId(shardId)
+            .dataCategory(dataCategory)
+            .dataType(dataType)
+            .build();
+        result = HASHED_INFIX.path(pathInput, FNV_1A_BASE64);
+        expected = derivePath(basePath, pathInput);
+        actual = result.buildAsString();
+        assertTrue(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual.startsWith(expected));
+
+        // assert with exact value for known base path
+        pathInput = PathInput.builder()
+            .basePath(fixedBlobPath)
+            .indexUUID(fixedIndexUUID)
+            .shardId(fixedShardId)
+            .dataCategory(dataCategory)
+            .dataType(dataType)
+            .build();
+        result = HASHED_INFIX.path(pathInput, FNV_1A_BASE64);
+        expected = "xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/AUBRfCIuWdk/k2ijhe877d7yuhx7/10/segments/data/";
+        actual = result.buildAsString();
+        assertTrue(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual.startsWith(expected));
+
+        // Segment Metadata
+        dataType = METADATA;
+        pathInput = PathInput.builder()
+            .basePath(blobPath)
+            .indexUUID(indexUUID)
+            .shardId(shardId)
+            .dataCategory(dataCategory)
+            .dataType(dataType)
+            .build();
+        result = HASHED_INFIX.path(pathInput, FNV_1A_BASE64);
+        expected = derivePath(basePath, pathInput);
+        actual = result.buildAsString();
+        assertTrue(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual.startsWith(expected));
+
+        // assert with exact value for known base path
+        pathInput = PathInput.builder()
+            .basePath(fixedBlobPath)
+            .indexUUID(fixedIndexUUID)
+            .shardId(fixedShardId)
+            .dataCategory(dataCategory)
+            .dataType(dataType)
+            .build();
+        result = HASHED_INFIX.path(pathInput, FNV_1A_BASE64);
+        expected = "xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/erwR-G735Uw/k2ijhe877d7yuhx7/10/segments/metadata/";
+        actual = result.buildAsString();
+        assertTrue(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual.startsWith(expected));
+
+        // Segment Lockfiles
+        dataType = LOCK_FILES;
+        pathInput = PathInput.builder()
+            .basePath(blobPath)
+            .indexUUID(indexUUID)
+            .shardId(shardId)
+            .dataCategory(dataCategory)
+            .dataType(dataType)
+            .build();
+        result = HASHED_INFIX.path(pathInput, FNV_1A_BASE64);
+        expected = derivePath(basePath, pathInput);
+        actual = result.buildAsString();
+        assertTrue(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual.startsWith(expected));
+
+        // assert with exact value for known base path
+        pathInput = PathInput.builder()
+            .basePath(fixedBlobPath)
+            .indexUUID(fixedIndexUUID)
+            .shardId(fixedShardId)
+            .dataCategory(dataCategory)
+            .dataType(dataType)
+            .build();
+        result = HASHED_INFIX.path(pathInput, FNV_1A_BASE64);
+        expected = "xjsdhj/ddjsha/yudy7sd/32hdhua7/89jdij/KeYDIk0mJXI/k2ijhe877d7yuhx7/10/segments/lock_files/";
+        actual = result.buildAsString();
+        assertTrue(new ParameterizedMessage("expected={} actual={}", expected, actual).getFormattedMessage(), actual.startsWith(expected));
+    }
+
+    private String derivePath(String basePath, PathInput pathInput) {
+        return "".equals(basePath)
+            ? String.join(
+                SEPARATOR,
+                FNV_1A_BASE64.hash(pathInput),
+                pathInput.indexUUID(),
+                pathInput.shardId(),
+                pathInput.dataCategory().getName(),
+                pathInput.dataType().getName()
+            )
+            : String.join(
+                SEPARATOR,
+                basePath,
+                FNV_1A_BASE64.hash(pathInput),
+                pathInput.indexUUID(),
+                pathInput.shardId(),
+                pathInput.dataCategory().getName(),
+                pathInput.dataType().getName()
+            );
+    }
+
+    private List<String> getPathList() {
+        List<String> pathList = new ArrayList<>();
+        int length = randomIntBetween(0, 5);
+        for (int i = 0; i < length; i++) {
+            pathList.add(randomAlphaOfLength(randomIntBetween(2, 5)));
+        }
+        return pathList;
+    }
+
+    private String getPath(List<String> pathList) {
+        String p = String.join(SEPARATOR, pathList);
+        if (p.isEmpty() || p.endsWith(SEPARATOR)) {
+            return p;
+        }
+        return p + SEPARATOR;
+    }
+}
diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteStorePathStrategyResolverTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteStorePathStrategyResolverTests.java
new file mode 100644
index 0000000000000..de61c902bf13e
--- /dev/null
+++ b/server/src/test/java/org/opensearch/index/remote/RemoteStorePathStrategyResolverTests.java
@@ -0,0 +1,148 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.remote;
+
+import org.opensearch.Version;
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm;
+import org.opensearch.index.remote.RemoteStoreEnums.PathType;
+import org.opensearch.indices.RemoteStoreSettings;
+import org.opensearch.test.OpenSearchTestCase;
+
+import static org.opensearch.indices.RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING;
+import static org.opensearch.indices.RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING;
+
+public class RemoteStorePathStrategyResolverTests extends OpenSearchTestCase {
+
+    public void testGetMinVersionOlder() {
+        Settings settings = Settings.builder().put(CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), randomFrom(PathType.values())).build();
+        ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        RemoteStoreSettings remoteStoreSettings = new RemoteStoreSettings(settings, clusterSettings);
+        RemoteStorePathStrategyResolver resolver = new RemoteStorePathStrategyResolver(remoteStoreSettings, () -> Version.V_2_13_0);
+        assertEquals(PathType.FIXED, resolver.get().getType());
+        assertNull(resolver.get().getHashAlgorithm());
+    }
+
+    public void testGetMinVersionNewer() {
+        PathType pathType = randomFrom(PathType.values());
+        Settings settings = Settings.builder().put(CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), pathType).build();
+        ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        RemoteStoreSettings remoteStoreSettings = new RemoteStoreSettings(settings, clusterSettings);
+        RemoteStorePathStrategyResolver resolver = new RemoteStorePathStrategyResolver(remoteStoreSettings, () -> Version.V_2_14_0);
+        assertEquals(pathType, resolver.get().getType());
+        if (pathType.requiresHashAlgorithm()) {
+            assertNotNull(resolver.get().getHashAlgorithm());
+        } else {
+            assertNull(resolver.get().getHashAlgorithm());
+        }
+    }
+
+    public void testGetStrategy() {
+        // FIXED type
+        Settings settings = Settings.builder().put(CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), PathType.FIXED).build();
+        ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        RemoteStoreSettings remoteStoreSettings = new RemoteStoreSettings(settings, clusterSettings);
+        RemoteStorePathStrategyResolver resolver = new RemoteStorePathStrategyResolver(remoteStoreSettings, () -> Version.V_2_14_0);
+        assertEquals(PathType.FIXED, resolver.get().getType());
+
+        // FIXED type with hash algorithm
+        settings = Settings.builder()
+            .put(CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), PathType.FIXED)
+            .put(CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING.getKey(), randomFrom(PathHashAlgorithm.values()))
+            .build();
+        clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        remoteStoreSettings = new RemoteStoreSettings(settings, clusterSettings);
+        resolver = new RemoteStorePathStrategyResolver(remoteStoreSettings, () -> Version.V_2_14_0);
+        assertEquals(PathType.FIXED, resolver.get().getType());
+
+        // HASHED_PREFIX type with FNV_1A_COMPOSITE
+        settings = Settings.builder().put(CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), PathType.HASHED_PREFIX).build();
+        clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        remoteStoreSettings = new RemoteStoreSettings(settings, clusterSettings);
+        resolver = new RemoteStorePathStrategyResolver(remoteStoreSettings, () -> Version.V_2_14_0);
+        assertEquals(PathType.HASHED_PREFIX, resolver.get().getType());
+        assertEquals(PathHashAlgorithm.FNV_1A_COMPOSITE_1, resolver.get().getHashAlgorithm());
+
+        // HASHED_PREFIX type with FNV_1A_COMPOSITE
+        settings = Settings.builder().put(CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), PathType.HASHED_PREFIX).build();
+        clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        remoteStoreSettings = new RemoteStoreSettings(settings, clusterSettings);
+        resolver = new RemoteStorePathStrategyResolver(remoteStoreSettings, () -> Version.V_2_14_0);
+        assertEquals(PathType.HASHED_PREFIX, resolver.get().getType());
+        assertEquals(PathHashAlgorithm.FNV_1A_COMPOSITE_1, resolver.get().getHashAlgorithm());
+
+        // HASHED_PREFIX type with FNV_1A_BASE64
+        settings = Settings.builder()
+            .put(CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), PathType.HASHED_PREFIX)
+            .put(CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING.getKey(), PathHashAlgorithm.FNV_1A_BASE64)
+            .build();
+        clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        remoteStoreSettings = new RemoteStoreSettings(settings, clusterSettings);
+        resolver = new RemoteStorePathStrategyResolver(remoteStoreSettings, () -> Version.V_2_14_0);
+        assertEquals(PathType.HASHED_PREFIX, resolver.get().getType());
+        assertEquals(PathHashAlgorithm.FNV_1A_BASE64, resolver.get().getHashAlgorithm());
+
+        // HASHED_PREFIX type with FNV_1A_BASE64
+        settings = Settings.builder()
+            .put(CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), PathType.HASHED_PREFIX)
+            .put(CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING.getKey(), PathHashAlgorithm.FNV_1A_BASE64)
+            .build();
+        clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        remoteStoreSettings = new RemoteStoreSettings(settings, clusterSettings);
+        resolver = new RemoteStorePathStrategyResolver(remoteStoreSettings, () -> Version.V_2_14_0);
+        assertEquals(PathType.HASHED_PREFIX, resolver.get().getType());
+        assertEquals(PathHashAlgorithm.FNV_1A_BASE64, resolver.get().getHashAlgorithm());
+    }
+
+    public void testGetStrategyWithDynamicUpdate() {
+
+        // Default value
+        Settings settings = Settings.builder().build();
+        ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+        RemoteStoreSettings remoteStoreSettings = new RemoteStoreSettings(settings, clusterSettings);
+        RemoteStorePathStrategyResolver resolver = new RemoteStorePathStrategyResolver(remoteStoreSettings, () -> Version.V_2_14_0);
+        assertEquals(PathType.FIXED, resolver.get().getType());
+        assertNull(resolver.get().getHashAlgorithm());
+
+        // Set HASHED_PREFIX with default hash algorithm
+        clusterSettings.applySettings(
+            Settings.builder().put(CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), PathType.HASHED_PREFIX).build()
+        );
+        assertEquals(PathType.HASHED_PREFIX, resolver.get().getType());
+        assertEquals(PathHashAlgorithm.FNV_1A_COMPOSITE_1, resolver.get().getHashAlgorithm());
+
+        // Set HASHED_PREFIX with FNV_1A_BASE64 hash algorithm
+        clusterSettings.applySettings(
+            Settings.builder()
+                .put(CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), PathType.HASHED_PREFIX)
+                .put(CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING.getKey(), PathHashAlgorithm.FNV_1A_BASE64)
+                .build()
+        );
+        assertEquals(PathType.HASHED_PREFIX, resolver.get().getType());
+        assertEquals(PathHashAlgorithm.FNV_1A_BASE64, resolver.get().getHashAlgorithm());
+
+        // Set HASHED_INFIX with default hash algorithm
+        clusterSettings.applySettings(
+            Settings.builder().put(CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), PathType.HASHED_INFIX).build()
+        );
+        assertEquals(PathType.HASHED_INFIX, resolver.get().getType());
+        assertEquals(PathHashAlgorithm.FNV_1A_COMPOSITE_1, resolver.get().getHashAlgorithm());
+
+        // Set HASHED_INFIX with FNV_1A_BASE64 hash algorithm
+        clusterSettings.applySettings(
+            Settings.builder()
+                .put(CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), PathType.HASHED_INFIX)
+                .put(CLUSTER_REMOTE_STORE_PATH_HASH_ALGORITHM_SETTING.getKey(), PathHashAlgorithm.FNV_1A_BASE64)
+                .build()
+        );
+        assertEquals(PathType.HASHED_INFIX, resolver.get().getType());
+        assertEquals(PathHashAlgorithm.FNV_1A_BASE64, resolver.get().getHashAlgorithm());
+    }
+}
diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteStoreTestsHelper.java b/server/src/test/java/org/opensearch/index/remote/RemoteStoreTestsHelper.java
index e072d3037caad..043b4493e8989 100644
--- a/server/src/test/java/org/opensearch/index/remote/RemoteStoreTestsHelper.java
+++ b/server/src/test/java/org/opensearch/index/remote/RemoteStoreTestsHelper.java
@@ -10,6 +10,7 @@
 
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.core.index.Index;
 import org.opensearch.core.index.shard.ShardId;
 import org.opensearch.index.IndexSettings;
 import org.opensearch.index.shard.IndexShard;
@@ -37,4 +38,22 @@ static IndexShard createIndexShard(ShardId shardId, boolean remoteStoreEnabled)
         when(indexShard.store()).thenReturn(store);
         return indexShard;
     }
+
+    public static IndexSettings createIndexSettings(boolean remote) {
+        return createIndexSettings(remote, Settings.EMPTY);
+    }
+
+    public static IndexSettings createIndexSettings(boolean remote, Settings settings) {
+        IndexSettings indexSettings;
+        if (remote) {
+            Settings nodeSettings = Settings.builder()
+                .put("node.name", "xyz")
+                .put("node.attr.remote_store.translog.repository", "seg_repo")
+                .build();
+            indexSettings = IndexSettingsModule.newIndexSettings(new Index("test_index", "_na_"), settings, nodeSettings);
+        } else {
+            indexSettings = IndexSettingsModule.newIndexSettings("test_index", settings);
+        }
+        return indexSettings;
+    }
 }
diff --git a/server/src/test/java/org/opensearch/index/remote/RemoteStoreUtilsTests.java b/server/src/test/java/org/opensearch/index/remote/RemoteStoreUtilsTests.java
index d3c7d754d6b61..c1fc0cdaa0d3b 100644
--- a/server/src/test/java/org/opensearch/index/remote/RemoteStoreUtilsTests.java
+++ b/server/src/test/java/org/opensearch/index/remote/RemoteStoreUtilsTests.java
@@ -8,24 +8,47 @@
 
 package org.opensearch.index.remote;
 
+import org.opensearch.cluster.node.DiscoveryNodes;
 import org.opensearch.common.blobstore.BlobMetadata;
 import org.opensearch.common.blobstore.support.PlainBlobMetadata;
+import org.opensearch.index.shard.IndexShardTestUtils;
 import org.opensearch.index.store.RemoteSegmentStoreDirectory;
 import org.opensearch.index.translog.transfer.TranslogTransferMetadata;
+import org.opensearch.node.remotestore.RemoteStoreNodeAttribute;
 import org.opensearch.test.OpenSearchTestCase;
 
+import java.math.BigInteger;
 import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Map;
 import java.util.stream.Collectors;
 
+import static org.opensearch.index.remote.RemoteStoreUtils.URL_BASE64_CHARSET;
+import static org.opensearch.index.remote.RemoteStoreUtils.longToCompositeBase64AndBinaryEncoding;
+import static org.opensearch.index.remote.RemoteStoreUtils.longToUrlBase64;
+import static org.opensearch.index.remote.RemoteStoreUtils.urlBase64ToLong;
 import static org.opensearch.index.remote.RemoteStoreUtils.verifyNoMultipleWriters;
+import static org.opensearch.index.shard.IndexShardTestUtils.MOCK_SEGMENT_REPO_NAME;
+import static org.opensearch.index.shard.IndexShardTestUtils.MOCK_TLOG_REPO_NAME;
 import static org.opensearch.index.store.RemoteSegmentStoreDirectory.MetadataFilenameUtils.METADATA_PREFIX;
 import static org.opensearch.index.store.RemoteSegmentStoreDirectory.MetadataFilenameUtils.SEPARATOR;
 import static org.opensearch.index.translog.transfer.TranslogTransferMetadata.METADATA_SEPARATOR;
 
 public class RemoteStoreUtilsTests extends OpenSearchTestCase {
 
+    private static Map<Character, Integer> BASE64_CHARSET_IDX_MAP;
+
+    static {
+        Map<Character, Integer> charToIndexMap = new HashMap<>();
+        for (int i = 0; i < URL_BASE64_CHARSET.length; i++) {
+            charToIndexMap.put(URL_BASE64_CHARSET[i], i);
+        }
+        BASE64_CHARSET_IDX_MAP = Collections.unmodifiableMap(charToIndexMap);
+    }
+
     private final String metadataFilename = RemoteSegmentStoreDirectory.MetadataFilenameUtils.getMetadataFilename(
         12,
         23,
@@ -179,4 +202,143 @@ public void testVerifyMultipleWriters_Translog() throws InterruptedException {
         );
     }
 
+    public void testLongToBase64() {
+        Map<Long, String> longToExpectedBase64String = Map.of(
+            -5537941589147079860L,
+            "syVHd0gGq0w",
+            -5878421770170594047L,
+            "rmumi5UPDQE",
+            -5147010836697060622L,
+            "uJIk6f-V6vI",
+            937096430362711837L,
+            "DQE8PQwOVx0",
+            8422273604115462710L,
+            "dOHtOEZzejY",
+            -2528761975013221124L,
+            "3OgIYbXSXPw",
+            -5512387536280560513L,
+            "s4AQvdu03H8",
+            -5749656451579835857L,
+            "sDUd65cNCi8",
+            5569654857969679538L,
+            "TUtjlYLPvLI",
+            -1563884000447039930L,
+            "6kv3yZNv9kY"
+        );
+        for (Map.Entry<Long, String> entry : longToExpectedBase64String.entrySet()) {
+            String base64Str = longToUrlBase64(entry.getKey());
+            assertEquals(entry.getValue(), base64Str);
+            assertEquals(11, entry.getValue().length());
+            assertEquals((long) entry.getKey(), urlBase64ToLong(base64Str));
+        }
+
+        int iters = randomInt(100);
+        for (int i = 0; i < iters; i++) {
+            long value = randomLong();
+            String base64Str = longToUrlBase64(value);
+            assertEquals(value, urlBase64ToLong(base64Str));
+        }
+    }
+
+    public void testLongToCompositeUrlBase64AndBinaryEncodingUsing20Bits() {
+        Map<Long, String> longToExpectedBase64String = Map.of(
+            -5537941589147079860L,
+            "s11001001010100",
+            -5878421770170594047L,
+            "r10011010111010",
+            -5147010836697060622L,
+            "u00100100100010",
+            937096430362711837L,
+            "D01000000010011",
+            8422273604115462710L,
+            "d00111000011110",
+            -2528761975013221124L,
+            "300111010000000",
+            -5512387536280560513L,
+            "s11100000000001",
+            -5749656451579835857L,
+            "s00001101010001",
+            5569654857969679538L,
+            "T01010010110110",
+            -1563884000447039930L,
+            "610010010111111"
+        );
+        for (Map.Entry<Long, String> entry : longToExpectedBase64String.entrySet()) {
+            String base64Str = RemoteStoreUtils.longToCompositeBase64AndBinaryEncoding(entry.getKey(), 20);
+            assertEquals(entry.getValue(), base64Str);
+            assertEquals(15, entry.getValue().length());
+            assertEquals(longToUrlBase64(entry.getKey()).charAt(0), base64Str.charAt(0));
+        }
+
+        int iters = randomInt(1000);
+        for (int i = 0; i < iters; i++) {
+            long value = randomLong();
+            assertEquals(RemoteStoreUtils.longToCompositeBase64AndBinaryEncoding(value, 20).charAt(0), longToUrlBase64(value).charAt(0));
+        }
+    }
+
+    public void testLongToCompositeUrlBase64AndBinaryEncoding() {
+        Map<Long, String> longToExpectedBase64String = Map.of(
+            -5537941589147079860L,
+            "s1100100101010001110111011101001000000001101010101101001100",
+            -5878421770170594047L,
+            "r1001101011101001101000101110010101000011110000110100000001",
+            -5147010836697060622L,
+            "u0010010010001001001110100111111111100101011110101011110010",
+            937096430362711837L,
+            "D0100000001001111000011110100001100000011100101011100011101",
+            8422273604115462710L,
+            "d0011100001111011010011100001000110011100110111101000110110",
+            -2528761975013221124L,
+            "30011101000000010000110000110110101110100100101110011111100",
+            -5512387536280560513L,
+            "s1110000000000100001011110111011011101101001101110001111111",
+            -5749656451579835857L,
+            "s0000110101000111011110101110010111000011010000101000101111",
+            5569654857969679538L,
+            "T0101001011011000111001010110000010110011111011110010110010",
+            -1563884000447039930L,
+            "61001001011111101111100100110010011011011111111011001000110"
+        );
+        for (Map.Entry<Long, String> entry : longToExpectedBase64String.entrySet()) {
+            Long hashValue = entry.getKey();
+            String expectedCompositeEncoding = entry.getValue();
+            String actualCompositeEncoding = longToCompositeBase64AndBinaryEncoding(hashValue, 64);
+            assertEquals(expectedCompositeEncoding, actualCompositeEncoding);
+            assertEquals(59, expectedCompositeEncoding.length());
+            assertEquals(longToUrlBase64(entry.getKey()).charAt(0), actualCompositeEncoding.charAt(0));
+            assertEquals(RemoteStoreUtils.longToCompositeBase64AndBinaryEncoding(hashValue, 20), actualCompositeEncoding.substring(0, 15));
+
+            Long computedHashValue = compositeUrlBase64BinaryEncodingToLong(actualCompositeEncoding);
+            assertEquals(hashValue, computedHashValue);
+        }
+
+        int iters = randomInt(1000);
+        for (int i = 0; i < iters; i++) {
+            long value = randomLong();
+            String compositeEncoding = longToCompositeBase64AndBinaryEncoding(value, 64);
+            assertEquals(value, compositeUrlBase64BinaryEncodingToLong(compositeEncoding));
+        }
+    }
+
+    public void testGetRemoteStoreRepoNameWithRemoteNodes() {
+        DiscoveryNodes discoveryNodes = DiscoveryNodes.builder().add(IndexShardTestUtils.getFakeRemoteEnabledNode("1")).build();
+        Map<String, String> expected = new HashMap<>();
+        expected.put(RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, MOCK_SEGMENT_REPO_NAME);
+        expected.put(RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, MOCK_TLOG_REPO_NAME);
+        assertEquals(expected, RemoteStoreUtils.getRemoteStoreRepoName(discoveryNodes));
+    }
+
+    public void testGetRemoteStoreRepoNameWithDocrepNdoes() {
+        DiscoveryNodes discoveryNodes = DiscoveryNodes.builder().add(IndexShardTestUtils.getFakeDiscoNode("1")).build();
+        assertTrue(RemoteStoreUtils.getRemoteStoreRepoName(discoveryNodes).isEmpty());
+    }
+
+    static long compositeUrlBase64BinaryEncodingToLong(String encodedValue) {
+        char ch = encodedValue.charAt(0);
+        int base64BitsIntValue = BASE64_CHARSET_IDX_MAP.get(ch);
+        String base64PartBinary = Integer.toBinaryString(base64BitsIntValue);
+        String binaryString = base64PartBinary + encodedValue.substring(1);
+        return new BigInteger(binaryString, 2).longValue();
+    }
 }
diff --git a/server/src/test/java/org/opensearch/index/replication/RetentionLeasesReplicationTests.java b/server/src/test/java/org/opensearch/index/replication/RetentionLeasesReplicationTests.java
index 8c59e92a3fe8a..904c9a70e61e0 100644
--- a/server/src/test/java/org/opensearch/index/replication/RetentionLeasesReplicationTests.java
+++ b/server/src/test/java/org/opensearch/index/replication/RetentionLeasesReplicationTests.java
@@ -45,6 +45,7 @@
 import org.opensearch.index.seqno.RetentionLeaseUtils;
 import org.opensearch.index.seqno.RetentionLeases;
 import org.opensearch.index.shard.IndexShard;
+import org.opensearch.index.shard.IndexShardTestUtils;
 import org.opensearch.test.VersionUtils;
 
 import java.util.ArrayList;
@@ -182,7 +183,8 @@ public void testTurnOffTranslogRetentionAfterAllShardStarted() throws Exception
                     null,
                     1L,
                     group.getPrimary().getReplicationGroup().getInSyncAllocationIds(),
-                    group.getPrimary().getReplicationGroup().getRoutingTable()
+                    group.getPrimary().getReplicationGroup().getRoutingTable(),
+                    IndexShardTestUtils.getFakeDiscoveryNodes(shard.routingEntry())
                 );
             }
             group.syncGlobalCheckpoint();
diff --git a/server/src/test/java/org/opensearch/index/search/stats/SearchStatsTests.java b/server/src/test/java/org/opensearch/index/search/stats/SearchStatsTests.java
index 5656b77445772..594700ea60b3e 100644
--- a/server/src/test/java/org/opensearch/index/search/stats/SearchStatsTests.java
+++ b/server/src/test/java/org/opensearch/index/search/stats/SearchStatsTests.java
@@ -57,9 +57,9 @@ public void testShardLevelSearchGroupStats() throws Exception {
         // let's create two dummy search stats with groups
         Map<String, Stats> groupStats1 = new HashMap<>();
         Map<String, Stats> groupStats2 = new HashMap<>();
-        groupStats2.put("group1", new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1));
-        SearchStats searchStats1 = new SearchStats(new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1), 0, groupStats1);
-        SearchStats searchStats2 = new SearchStats(new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1), 0, groupStats2);
+        groupStats2.put("group1", new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1));
+        SearchStats searchStats1 = new SearchStats(new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1), 0, groupStats1);
+        SearchStats searchStats2 = new SearchStats(new Stats(1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1), 0, groupStats2);
 
         // adding these two search stats and checking group stats are correct
         searchStats1.add(searchStats2);
@@ -128,6 +128,7 @@ private static void assertStats(Stats stats, long equalTo) {
         assertEquals(equalTo, stats.getSuggestCount());
         assertEquals(equalTo, stats.getSuggestTimeInMillis());
         assertEquals(equalTo, stats.getSuggestCurrent());
+        assertEquals(equalTo, stats.getSearchIdleReactivateCount());
         // avg_concurrency is not summed up across stats
         assertEquals(1, stats.getConcurrentAvgSliceCount(), 0);
     }
diff --git a/server/src/test/java/org/opensearch/index/seqno/GlobalCheckpointSyncActionTests.java b/server/src/test/java/org/opensearch/index/seqno/GlobalCheckpointSyncActionTests.java
index 8363ea3757a2b..a27f3476888eb 100644
--- a/server/src/test/java/org/opensearch/index/seqno/GlobalCheckpointSyncActionTests.java
+++ b/server/src/test/java/org/opensearch/index/seqno/GlobalCheckpointSyncActionTests.java
@@ -54,6 +54,7 @@
 
 import java.util.Collections;
 
+import static org.opensearch.index.remote.RemoteStoreTestsHelper.createIndexSettings;
 import static org.opensearch.test.ClusterServiceUtils.createClusterService;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.never;
@@ -110,6 +111,7 @@ public void testTranslogSyncAfterGlobalCheckpointSync() throws Exception {
         final ShardId shardId = new ShardId(index, id);
         when(indexShard.shardId()).thenReturn(shardId);
 
+        when(indexShard.indexSettings()).thenReturn(createIndexSettings(false));
         final Translog.Durability durability = randomFrom(Translog.Durability.ASYNC, Translog.Durability.REQUEST);
         when(indexShard.getTranslogDurability()).thenReturn(durability);
 
@@ -158,14 +160,14 @@ public void testTranslogSyncAfterGlobalCheckpointSync() throws Exception {
     public void testGetReplicationModeWithRemoteTranslog() {
         final GlobalCheckpointSyncAction action = createAction();
         final IndexShard indexShard = mock(IndexShard.class);
-        when(indexShard.isRemoteTranslogEnabled()).thenReturn(true);
+        when(indexShard.indexSettings()).thenReturn(createIndexSettings(true));
         assertEquals(ReplicationMode.NO_REPLICATION, action.getReplicationMode(indexShard));
     }
 
     public void testGetReplicationModeWithLocalTranslog() {
         final GlobalCheckpointSyncAction action = createAction();
         final IndexShard indexShard = mock(IndexShard.class);
-        when(indexShard.isRemoteTranslogEnabled()).thenReturn(false);
+        when(indexShard.indexSettings()).thenReturn(createIndexSettings(false));
         assertEquals(ReplicationMode.FULL_REPLICATION, action.getReplicationMode(indexShard));
     }
 
@@ -191,6 +193,7 @@ public void testMayBeSyncTranslogWithRemoteTranslog() throws Exception {
         when(indexShard.getLastKnownGlobalCheckpoint()).thenReturn(globalCheckpoint);
         when(indexShard.getLastSyncedGlobalCheckpoint()).thenReturn(globalCheckpoint - 1);
         when(indexShard.getTranslogDurability()).thenReturn(Translog.Durability.REQUEST);
+        when(indexShard.indexSettings()).thenReturn(createIndexSettings(true));
 
         action.shardOperationOnPrimary(primaryRequest, indexShard, ActionTestUtils.assertNoFailureListener(r -> {}));
         verify(indexShard, never()).sync();
@@ -205,6 +208,7 @@ public void testMayBeSyncTranslogWithLocalTranslog() throws Exception {
         when(indexShard.getLastKnownGlobalCheckpoint()).thenReturn(globalCheckpoint);
         when(indexShard.getLastSyncedGlobalCheckpoint()).thenReturn(globalCheckpoint - 1);
         when(indexShard.getTranslogDurability()).thenReturn(Translog.Durability.REQUEST);
+        when(indexShard.indexSettings()).thenReturn(createIndexSettings(false));
 
         action.shardOperationOnPrimary(primaryRequest, indexShard, ActionTestUtils.assertNoFailureListener(r -> {}));
         verify(indexShard).sync();
diff --git a/server/src/test/java/org/opensearch/index/seqno/PeerRecoveryRetentionLeaseExpiryTests.java b/server/src/test/java/org/opensearch/index/seqno/PeerRecoveryRetentionLeaseExpiryTests.java
index ca80c7b9c4884..7a9f1d7baa12e 100644
--- a/server/src/test/java/org/opensearch/index/seqno/PeerRecoveryRetentionLeaseExpiryTests.java
+++ b/server/src/test/java/org/opensearch/index/seqno/PeerRecoveryRetentionLeaseExpiryTests.java
@@ -93,7 +93,8 @@ public void setUpReplicationTracker() throws InterruptedException {
             value -> {},
             currentTimeMillis::get,
             (leases, listener) -> {},
-            () -> safeCommitInfo
+            () -> safeCommitInfo,
+            sId -> false
         );
         replicationTracker.updateFromClusterManager(
             1L,
diff --git a/server/src/test/java/org/opensearch/index/seqno/ReplicationTrackerRetentionLeaseTests.java b/server/src/test/java/org/opensearch/index/seqno/ReplicationTrackerRetentionLeaseTests.java
index 3cd60ac973709..fdbe89422a2aa 100644
--- a/server/src/test/java/org/opensearch/index/seqno/ReplicationTrackerRetentionLeaseTests.java
+++ b/server/src/test/java/org/opensearch/index/seqno/ReplicationTrackerRetentionLeaseTests.java
@@ -84,7 +84,8 @@ public void testAddOrRenewRetentionLease() {
             value -> {},
             () -> 0L,
             (leases, listener) -> {},
-            OPS_BASED_RECOVERY_ALWAYS_REASONABLE
+            OPS_BASED_RECOVERY_ALWAYS_REASONABLE,
+            NON_REMOTE_DISCOVERY_NODE
         );
         replicationTracker.updateFromClusterManager(
             randomNonNegativeLong(),
@@ -132,7 +133,8 @@ public void testAddDuplicateRetentionLease() {
             value -> {},
             () -> 0L,
             (leases, listener) -> {},
-            OPS_BASED_RECOVERY_ALWAYS_REASONABLE
+            OPS_BASED_RECOVERY_ALWAYS_REASONABLE,
+            NON_REMOTE_DISCOVERY_NODE
         );
         replicationTracker.updateFromClusterManager(
             randomNonNegativeLong(),
@@ -164,7 +166,8 @@ public void testRenewNotFoundRetentionLease() {
             value -> {},
             () -> 0L,
             (leases, listener) -> {},
-            OPS_BASED_RECOVERY_ALWAYS_REASONABLE
+            OPS_BASED_RECOVERY_ALWAYS_REASONABLE,
+            NON_REMOTE_DISCOVERY_NODE
         );
         replicationTracker.updateFromClusterManager(
             randomNonNegativeLong(),
@@ -202,7 +205,8 @@ public void testAddRetentionLeaseCausesRetentionLeaseSync() {
                     equalTo(retainingSequenceNumbers)
                 );
             },
-            OPS_BASED_RECOVERY_ALWAYS_REASONABLE
+            OPS_BASED_RECOVERY_ALWAYS_REASONABLE,
+            NON_REMOTE_DISCOVERY_NODE
         );
         reference.set(replicationTracker);
         replicationTracker.updateFromClusterManager(
@@ -241,7 +245,8 @@ public void testRemoveRetentionLease() {
             value -> {},
             () -> 0L,
             (leases, listener) -> {},
-            OPS_BASED_RECOVERY_ALWAYS_REASONABLE
+            OPS_BASED_RECOVERY_ALWAYS_REASONABLE,
+            NON_REMOTE_DISCOVERY_NODE
         );
         replicationTracker.updateFromClusterManager(
             randomNonNegativeLong(),
@@ -305,7 +310,8 @@ public void testCloneRetentionLease() {
                 assertTrue(synced.compareAndSet(false, true));
                 listener.onResponse(new ReplicationResponse());
             },
-            OPS_BASED_RECOVERY_ALWAYS_REASONABLE
+            OPS_BASED_RECOVERY_ALWAYS_REASONABLE,
+            NON_REMOTE_DISCOVERY_NODE
         );
         replicationTrackerRef.set(replicationTracker);
         replicationTracker.updateFromClusterManager(
@@ -351,7 +357,8 @@ public void testCloneNonexistentRetentionLease() {
             value -> {},
             () -> 0L,
             (leases, listener) -> {},
-            OPS_BASED_RECOVERY_ALWAYS_REASONABLE
+            OPS_BASED_RECOVERY_ALWAYS_REASONABLE,
+            NON_REMOTE_DISCOVERY_NODE
         );
         replicationTracker.updateFromClusterManager(
             randomNonNegativeLong(),
@@ -380,7 +387,8 @@ public void testCloneDuplicateRetentionLease() {
             value -> {},
             () -> 0L,
             (leases, listener) -> {},
-            OPS_BASED_RECOVERY_ALWAYS_REASONABLE
+            OPS_BASED_RECOVERY_ALWAYS_REASONABLE,
+            NON_REMOTE_DISCOVERY_NODE
         );
         replicationTracker.updateFromClusterManager(
             randomNonNegativeLong(),
@@ -413,7 +421,8 @@ public void testRemoveNotFound() {
             value -> {},
             () -> 0L,
             (leases, listener) -> {},
-            OPS_BASED_RECOVERY_ALWAYS_REASONABLE
+            OPS_BASED_RECOVERY_ALWAYS_REASONABLE,
+            NON_REMOTE_DISCOVERY_NODE
         );
         replicationTracker.updateFromClusterManager(
             randomNonNegativeLong(),
@@ -451,7 +460,8 @@ public void testRemoveRetentionLeaseCausesRetentionLeaseSync() {
                     equalTo(retainingSequenceNumbers)
                 );
             },
-            OPS_BASED_RECOVERY_ALWAYS_REASONABLE
+            OPS_BASED_RECOVERY_ALWAYS_REASONABLE,
+            NON_REMOTE_DISCOVERY_NODE
         );
         reference.set(replicationTracker);
         replicationTracker.updateFromClusterManager(
@@ -504,7 +514,8 @@ private void runExpirationTest(final boolean primaryMode) {
             value -> {},
             currentTimeMillis::get,
             (leases, listener) -> {},
-            OPS_BASED_RECOVERY_ALWAYS_REASONABLE
+            OPS_BASED_RECOVERY_ALWAYS_REASONABLE,
+            NON_REMOTE_DISCOVERY_NODE
         );
         replicationTracker.updateFromClusterManager(
             randomNonNegativeLong(),
@@ -583,7 +594,8 @@ public void testReplicaIgnoresOlderRetentionLeasesVersion() {
             value -> {},
             () -> 0L,
             (leases, listener) -> {},
-            OPS_BASED_RECOVERY_ALWAYS_REASONABLE
+            OPS_BASED_RECOVERY_ALWAYS_REASONABLE,
+            NON_REMOTE_DISCOVERY_NODE
         );
         replicationTracker.updateFromClusterManager(
             randomNonNegativeLong(),
@@ -636,7 +648,8 @@ public void testLoadAndPersistRetentionLeases() throws IOException {
             value -> {},
             () -> 0L,
             (leases, listener) -> {},
-            OPS_BASED_RECOVERY_ALWAYS_REASONABLE
+            OPS_BASED_RECOVERY_ALWAYS_REASONABLE,
+            NON_REMOTE_DISCOVERY_NODE
         );
         replicationTracker.updateFromClusterManager(
             randomNonNegativeLong(),
@@ -671,7 +684,8 @@ public void testUnnecessaryPersistenceOfRetentionLeases() throws IOException {
             value -> {},
             () -> 0L,
             (leases, listener) -> {},
-            OPS_BASED_RECOVERY_ALWAYS_REASONABLE
+            OPS_BASED_RECOVERY_ALWAYS_REASONABLE,
+            NON_REMOTE_DISCOVERY_NODE
         );
         replicationTracker.updateFromClusterManager(
             randomNonNegativeLong(),
@@ -724,7 +738,8 @@ public void testPersistRetentionLeasesUnderConcurrency() throws IOException {
             value -> {},
             () -> 0L,
             (leases, listener) -> {},
-            OPS_BASED_RECOVERY_ALWAYS_REASONABLE
+            OPS_BASED_RECOVERY_ALWAYS_REASONABLE,
+            NON_REMOTE_DISCOVERY_NODE
         );
         replicationTracker.updateFromClusterManager(
             randomNonNegativeLong(),
@@ -788,7 +803,8 @@ public void testRenewLeaseWithLowerRetainingSequenceNumber() throws Exception {
             value -> {},
             () -> 0L,
             (leases, listener) -> {},
-            OPS_BASED_RECOVERY_ALWAYS_REASONABLE
+            OPS_BASED_RECOVERY_ALWAYS_REASONABLE,
+            NON_REMOTE_DISCOVERY_NODE
         );
         replicationTracker.updateFromClusterManager(
             randomNonNegativeLong(),
diff --git a/server/src/test/java/org/opensearch/index/seqno/ReplicationTrackerTestCase.java b/server/src/test/java/org/opensearch/index/seqno/ReplicationTrackerTestCase.java
index e61d27695a5e5..daeefeff59c94 100644
--- a/server/src/test/java/org/opensearch/index/seqno/ReplicationTrackerTestCase.java
+++ b/server/src/test/java/org/opensearch/index/seqno/ReplicationTrackerTestCase.java
@@ -40,11 +40,13 @@
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.index.shard.ShardId;
 import org.opensearch.index.engine.SafeCommitInfo;
+import org.opensearch.index.remote.RemoteStoreTestsHelper;
 import org.opensearch.test.IndexSettingsModule;
 import org.opensearch.test.OpenSearchTestCase;
 
 import java.util.Collections;
 import java.util.Set;
+import java.util.function.Function;
 import java.util.function.LongConsumer;
 import java.util.function.LongSupplier;
 import java.util.function.Supplier;
@@ -57,18 +59,20 @@ ReplicationTracker newTracker(
         final AllocationId allocationId,
         final LongConsumer updatedGlobalCheckpoint,
         final LongSupplier currentTimeMillisSupplier,
-        final Settings settings
+        final Settings settings,
+        final boolean remote
     ) {
         return new ReplicationTracker(
             new ShardId("test", "_na_", 0),
             allocationId.getId(),
-            IndexSettingsModule.newIndexSettings("test", settings),
+            remote ? RemoteStoreTestsHelper.createIndexSettings(true, settings) : IndexSettingsModule.newIndexSettings("test", settings),
             randomNonNegativeLong(),
             UNASSIGNED_SEQ_NO,
             updatedGlobalCheckpoint,
             currentTimeMillisSupplier,
             (leases, listener) -> {},
-            OPS_BASED_RECOVERY_ALWAYS_REASONABLE
+            OPS_BASED_RECOVERY_ALWAYS_REASONABLE,
+            remote ? REMOTE_DISCOVERY_NODE : NON_REMOTE_DISCOVERY_NODE
         );
     }
 
@@ -80,8 +84,21 @@ ReplicationTracker newTracker(
         return newTracker(allocationId, updatedGlobalCheckpoint, currentTimeMillisSupplier, Settings.EMPTY);
     }
 
+    ReplicationTracker newTracker(
+        final AllocationId allocationId,
+        final LongConsumer updatedGlobalCheckpoint,
+        final LongSupplier currentTimeMillisSupplier,
+        final Settings settings
+    ) {
+        return newTracker(allocationId, updatedGlobalCheckpoint, currentTimeMillisSupplier, settings, false);
+    }
+
     static final Supplier<SafeCommitInfo> OPS_BASED_RECOVERY_ALWAYS_REASONABLE = () -> SafeCommitInfo.EMPTY;
 
+    static final Function<String, Boolean> NON_REMOTE_DISCOVERY_NODE = shardId -> false;
+
+    static final Function<String, Boolean> REMOTE_DISCOVERY_NODE = shardId -> true;
+
     static String nodeIdFromAllocationId(final AllocationId allocationId) {
         return "n-" + allocationId.getId().substring(0, 8);
     }
diff --git a/server/src/test/java/org/opensearch/index/seqno/ReplicationTrackerTests.java b/server/src/test/java/org/opensearch/index/seqno/ReplicationTrackerTests.java
index 7971591e82bab..233a99cbe4a73 100644
--- a/server/src/test/java/org/opensearch/index/seqno/ReplicationTrackerTests.java
+++ b/server/src/test/java/org/opensearch/index/seqno/ReplicationTrackerTests.java
@@ -446,6 +446,10 @@ public void testWaitForAllocationIdToBeInSync() throws Exception {
 
     private AtomicLong updatedGlobalCheckpoint = new AtomicLong(UNASSIGNED_SEQ_NO);
 
+    private ReplicationTracker newTracker(final AllocationId allocationId, Settings settings, boolean remote) {
+        return newTracker(allocationId, updatedGlobalCheckpoint::set, () -> 0L, settings, remote);
+    }
+
     private ReplicationTracker newTracker(final AllocationId allocationId, Settings settings) {
         return newTracker(allocationId, updatedGlobalCheckpoint::set, () -> 0L, settings);
     }
@@ -759,7 +763,8 @@ public void testPrimaryContextHandoff() throws IOException {
             onUpdate,
             () -> 0L,
             onNewRetentionLease,
-            OPS_BASED_RECOVERY_ALWAYS_REASONABLE
+            OPS_BASED_RECOVERY_ALWAYS_REASONABLE,
+            NON_REMOTE_DISCOVERY_NODE
         );
         ReplicationTracker newPrimary = new ReplicationTracker(
             shardId,
@@ -770,7 +775,8 @@ public void testPrimaryContextHandoff() throws IOException {
             onUpdate,
             () -> 0L,
             onNewRetentionLease,
-            OPS_BASED_RECOVERY_ALWAYS_REASONABLE
+            OPS_BASED_RECOVERY_ALWAYS_REASONABLE,
+            NON_REMOTE_DISCOVERY_NODE
         );
 
         Set<String> allocationIds = new HashSet<>(Arrays.asList(oldPrimary.shardAllocationId, newPrimary.shardAllocationId));
@@ -1300,7 +1306,7 @@ public void testGlobalCheckpointUpdateWithRemoteTranslogEnabled() {
             .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT)
             .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true")
             .build();
-        final ReplicationTracker tracker = newTracker(primaryId, settings);
+        final ReplicationTracker tracker = newTracker(primaryId, settings, true);
         assertThat(tracker.getGlobalCheckpoint(), equalTo(UNASSIGNED_SEQ_NO));
 
         long primaryLocalCheckpoint = activeWithCheckpoints.get(primaryId);
@@ -1378,7 +1384,7 @@ public void testUpdateFromClusterManagerWithRemoteTranslogEnabled() {
             .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT)
             .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true")
             .build();
-        final ReplicationTracker tracker = newTracker(primaryId, settings);
+        final ReplicationTracker tracker = newTracker(primaryId, settings, true);
         assertThat(tracker.getGlobalCheckpoint(), equalTo(UNASSIGNED_SEQ_NO));
 
         long primaryLocalCheckpoint = activeWithCheckpoints.get(primaryId);
@@ -1476,7 +1482,7 @@ public void testMarkAllocationIdAsInSyncWithRemoteTranslogEnabled() throws Excep
             .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT)
             .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true")
             .build();
-        final ReplicationTracker tracker = newTracker(primaryId, settings);
+        final ReplicationTracker tracker = newTracker(primaryId, settings, true);
         tracker.updateFromClusterManager(initialClusterStateVersion, ids(active), routingTable(initializing, primaryId));
         final long localCheckpoint = randomLongBetween(0, Long.MAX_VALUE - 1);
         tracker.activatePrimaryMode(localCheckpoint);
@@ -1504,7 +1510,7 @@ public void testMissingActiveIdsDoesNotPreventAdvanceWithRemoteTranslogEnabled()
             .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT)
             .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true")
             .build();
-        final ReplicationTracker tracker = newTracker(primaryId, settings);
+        final ReplicationTracker tracker = newTracker(primaryId, settings, true);
         tracker.updateFromClusterManager(randomNonNegativeLong(), ids(active.keySet()), routingTable(initializing.keySet(), primaryId));
         tracker.activatePrimaryMode(NO_OPS_PERFORMED);
         List<AllocationId> initializingRandomSubset = randomSubsetOf(initializing.keySet());
@@ -1537,7 +1543,7 @@ public void testMissingInSyncIdsDoesNotPreventAdvanceWithRemoteTranslogEnabled()
             .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT)
             .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true")
             .build();
-        final ReplicationTracker tracker = newTracker(primaryId, settings);
+        final ReplicationTracker tracker = newTracker(primaryId, settings, true);
         tracker.updateFromClusterManager(randomNonNegativeLong(), ids(active.keySet()), routingTable(initializing.keySet(), primaryId));
         tracker.activatePrimaryMode(NO_OPS_PERFORMED);
         randomSubsetOf(randomIntBetween(1, initializing.size() - 1), initializing.keySet()).forEach(
@@ -1606,8 +1612,8 @@ public void testInSyncIdsAreRemovedIfNotValidatedByClusterManagerWithRemoteTrans
             .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT)
             .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true")
             .build();
-        final ReplicationTracker tracker = newTracker(primaryId, settings);
-        tracker.updateFromClusterManager(initialClusterStateVersion, ids(active), routingTable(initializing, primaryId));
+        final ReplicationTracker tracker = newTracker(primaryId, settings, true);
+        tracker.updateFromClusterManager(initialClusterStateVersion, ids(active), routingTable(initializing, active, primaryId));
         tracker.activatePrimaryMode(NO_OPS_PERFORMED);
         if (randomBoolean()) {
             initializingToStay.keySet().forEach(k -> markAsTrackingAndInSyncQuietly(tracker, k.getId(), NO_OPS_PERFORMED));
@@ -1655,7 +1661,7 @@ public void testUpdateAllocationIdsFromClusterManagerWithRemoteTranslogEnabled()
             .put(IndexMetadata.SETTING_REPLICATION_TYPE, ReplicationType.SEGMENT)
             .put(IndexMetadata.SETTING_REMOTE_STORE_ENABLED, "true")
             .build();
-        final ReplicationTracker tracker = newTracker(primaryId, settings);
+        final ReplicationTracker tracker = newTracker(primaryId, settings, true);
         tracker.updateFromClusterManager(initialClusterStateVersion, ids(activeAllocationIds), routingTable);
         tracker.activatePrimaryMode(NO_OPS_PERFORMED);
         assertThat(tracker.getReplicationGroup().getInSyncAllocationIds(), equalTo(ids(activeAllocationIds)));
@@ -2080,7 +2086,8 @@ public void testPrimaryContextHandoffWithRemoteTranslogEnabled() throws IOExcept
             onUpdate,
             () -> 0L,
             onNewRetentionLease,
-            OPS_BASED_RECOVERY_ALWAYS_REASONABLE
+            OPS_BASED_RECOVERY_ALWAYS_REASONABLE,
+            REMOTE_DISCOVERY_NODE
         );
         ReplicationTracker newPrimary = new ReplicationTracker(
             shardId,
@@ -2091,7 +2098,8 @@ public void testPrimaryContextHandoffWithRemoteTranslogEnabled() throws IOExcept
             onUpdate,
             () -> 0L,
             onNewRetentionLease,
-            OPS_BASED_RECOVERY_ALWAYS_REASONABLE
+            OPS_BASED_RECOVERY_ALWAYS_REASONABLE,
+            REMOTE_DISCOVERY_NODE
         );
 
         Set<String> allocationIds = new HashSet<>(Arrays.asList(oldPrimary.shardAllocationId, newPrimary.shardAllocationId));
diff --git a/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseBackgroundSyncActionTests.java b/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseBackgroundSyncActionTests.java
index ed04d9a20f18e..d5d7163b66698 100644
--- a/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseBackgroundSyncActionTests.java
+++ b/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseBackgroundSyncActionTests.java
@@ -60,6 +60,7 @@
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import static org.opensearch.index.remote.RemoteStoreTestsHelper.createIndexSettings;
 import static org.opensearch.test.ClusterServiceUtils.createClusterService;
 import static org.hamcrest.Matchers.sameInstance;
 import static org.mockito.Mockito.mock;
@@ -215,14 +216,14 @@ public void testBlocks() {
     public void testGetReplicationModeWithRemoteTranslog() {
         final RetentionLeaseBackgroundSyncAction action = createAction();
         final IndexShard indexShard = mock(IndexShard.class);
-        when(indexShard.isRemoteTranslogEnabled()).thenReturn(true);
+        when(indexShard.indexSettings()).thenReturn(createIndexSettings(true));
         assertEquals(ReplicationMode.NO_REPLICATION, action.getReplicationMode(indexShard));
     }
 
     public void testGetReplicationModeWithLocalTranslog() {
         final RetentionLeaseBackgroundSyncAction action = createAction();
         final IndexShard indexShard = mock(IndexShard.class);
-        when(indexShard.isRemoteTranslogEnabled()).thenReturn(false);
+        when(indexShard.indexSettings()).thenReturn(createIndexSettings(false));
         assertEquals(ReplicationMode.FULL_REPLICATION, action.getReplicationMode(indexShard));
     }
 
diff --git a/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseSyncActionTests.java b/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseSyncActionTests.java
index 63a9ac2f2e8ec..7610b8bc39296 100644
--- a/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseSyncActionTests.java
+++ b/server/src/test/java/org/opensearch/index/seqno/RetentionLeaseSyncActionTests.java
@@ -60,6 +60,7 @@
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import static java.util.Collections.emptyMap;
+import static org.opensearch.index.remote.RemoteStoreTestsHelper.createIndexSettings;
 import static org.opensearch.test.ClusterServiceUtils.createClusterService;
 import static org.hamcrest.Matchers.sameInstance;
 import static org.mockito.Mockito.mock;
@@ -215,15 +216,15 @@ public void testBlocks() {
     public void testGetReplicationModeWithRemoteTranslog() {
         final RetentionLeaseSyncAction action = createAction();
         final IndexShard indexShard = mock(IndexShard.class);
-        when(indexShard.isRemoteTranslogEnabled()).thenReturn(true);
+        when(indexShard.indexSettings()).thenReturn(createIndexSettings(true));
         assertEquals(ReplicationMode.NO_REPLICATION, action.getReplicationMode(indexShard));
     }
 
     public void testGetReplicationModeWithLocalTranslog() {
         final RetentionLeaseSyncAction action = createAction();
         final IndexShard indexShard = mock(IndexShard.class);
-        when(indexShard.isRemoteTranslogEnabled()).thenReturn(true);
-        assertEquals(ReplicationMode.NO_REPLICATION, action.getReplicationMode(indexShard));
+        when(indexShard.indexSettings()).thenReturn(createIndexSettings(false));
+        assertEquals(ReplicationMode.FULL_REPLICATION, action.getReplicationMode(indexShard));
     }
 
     private RetentionLeaseSyncAction createAction() {
diff --git a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java
index 537bfcf8f8a6b..e5bfa8caee79a 100644
--- a/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java
+++ b/server/src/test/java/org/opensearch/index/shard/IndexShardTests.java
@@ -697,7 +697,8 @@ public void testPrimaryPromotionRollsGeneration() throws Exception {
             (shard, listener) -> {},
             0L,
             Collections.singleton(primaryRouting.allocationId().getId()),
-            new IndexShardRoutingTable.Builder(primaryRouting.shardId()).addShard(primaryRouting).build()
+            new IndexShardRoutingTable.Builder(primaryRouting.shardId()).addShard(primaryRouting).build(),
+            IndexShardTestUtils.getFakeDiscoveryNodes(primaryRouting)
         );
 
         /*
@@ -764,7 +765,8 @@ public void testOperationPermitsOnPrimaryShards() throws Exception {
             },
                 0L,
                 Collections.singleton(indexShard.routingEntry().allocationId().getId()),
-                new IndexShardRoutingTable.Builder(indexShard.shardId()).addShard(primaryRouting).build()
+                new IndexShardRoutingTable.Builder(indexShard.shardId()).addShard(primaryRouting).build(),
+                IndexShardTestUtils.getFakeDiscoveryNodes(primaryRouting)
             );
             latch.await();
             assertThat(indexShard.getActiveOperationsCount(), is(oneOf(0, IndexShard.OPERATIONS_BLOCKED)));
@@ -1446,7 +1448,8 @@ public void onFailure(Exception e) {
             (s, r) -> resyncLatch.countDown(),
             1L,
             Collections.singleton(newRouting.allocationId().getId()),
-            new IndexShardRoutingTable.Builder(newRouting.shardId()).addShard(newRouting).build()
+            new IndexShardRoutingTable.Builder(newRouting.shardId()).addShard(newRouting).build(),
+            IndexShardTestUtils.getFakeDiscoveryNodes(newRouting)
         );
         resyncLatch.await();
         assertThat(indexShard.getLocalCheckpoint(), equalTo(maxSeqNo));
@@ -3284,7 +3287,7 @@ public void testRecoverFromTranslog() throws IOException {
         Translog.Snapshot snapshot = TestTranslog.newSnapshotFromOperations(operations);
         primary.markAsRecovering(
             "store",
-            new RecoveryState(primary.routingEntry(), getFakeDiscoNode(primary.routingEntry().currentNodeId()), null)
+            new RecoveryState(primary.routingEntry(), IndexShardTestUtils.getFakeDiscoNode(primary.routingEntry().currentNodeId()), null)
         );
         recoverFromStore(primary);
 
@@ -4029,15 +4032,19 @@ public void testReadSnapshotAndCheckIndexConcurrently() throws Exception {
         if (isPrimary) {
             newShard.markAsRecovering(
                 "store",
-                new RecoveryState(newShard.routingEntry(), getFakeDiscoNode(newShard.routingEntry().currentNodeId()), null)
+                new RecoveryState(
+                    newShard.routingEntry(),
+                    IndexShardTestUtils.getFakeDiscoNode(newShard.routingEntry().currentNodeId()),
+                    null
+                )
             );
         } else {
             newShard.markAsRecovering(
                 "peer",
                 new RecoveryState(
                     newShard.routingEntry(),
-                    getFakeDiscoNode(newShard.routingEntry().currentNodeId()),
-                    getFakeDiscoNode(newShard.routingEntry().currentNodeId())
+                    IndexShardTestUtils.getFakeDiscoNode(newShard.routingEntry().currentNodeId()),
+                    IndexShardTestUtils.getFakeDiscoNode(newShard.routingEntry().currentNodeId())
                 )
             );
         }
diff --git a/server/src/test/java/org/opensearch/index/shard/PrimaryReplicaSyncerTests.java b/server/src/test/java/org/opensearch/index/shard/PrimaryReplicaSyncerTests.java
index b1bcaac2c1947..09903a8b44cb5 100644
--- a/server/src/test/java/org/opensearch/index/shard/PrimaryReplicaSyncerTests.java
+++ b/server/src/test/java/org/opensearch/index/shard/PrimaryReplicaSyncerTests.java
@@ -111,7 +111,8 @@ public void testSyncerSendsOffCorrectDocuments() throws Exception {
             null,
             1000L,
             Collections.singleton(allocationId),
-            new IndexShardRoutingTable.Builder(shard.shardId()).addShard(shard.routingEntry()).build()
+            new IndexShardRoutingTable.Builder(shard.shardId()).addShard(shard.routingEntry()).build(),
+            IndexShardTestUtils.getFakeDiscoveryNodes(shard.routingEntry())
         );
         shard.updateLocalCheckpointForShard(allocationId, globalCheckPoint);
         assertEquals(globalCheckPoint, shard.getLastKnownGlobalCheckpoint());
@@ -190,7 +191,8 @@ public void testSyncerOnClosingShard() throws Exception {
             null,
             1000L,
             Collections.singleton(allocationId),
-            new IndexShardRoutingTable.Builder(shard.shardId()).addShard(shard.routingEntry()).build()
+            new IndexShardRoutingTable.Builder(shard.shardId()).addShard(shard.routingEntry()).build(),
+            IndexShardTestUtils.getFakeDiscoveryNodes(shard.routingEntry())
         );
 
         CountDownLatch syncCalledLatch = new CountDownLatch(1);
diff --git a/server/src/test/java/org/opensearch/index/shard/RefreshListenersTests.java b/server/src/test/java/org/opensearch/index/shard/RefreshListenersTests.java
index a45b25f04060b..8a77fbca2915d 100644
--- a/server/src/test/java/org/opensearch/index/shard/RefreshListenersTests.java
+++ b/server/src/test/java/org/opensearch/index/shard/RefreshListenersTests.java
@@ -134,7 +134,8 @@ public void setupListeners() throws Exception {
             createTempDir("translog"),
             indexSettings,
             BigArrays.NON_RECYCLING_INSTANCE,
-            ""
+            "",
+            false
         );
         Engine.EventListener eventListener = new Engine.EventListener() {
             @Override
diff --git a/server/src/test/java/org/opensearch/index/shard/ReleasableRetryableRefreshListenerTests.java b/server/src/test/java/org/opensearch/index/shard/ReleasableRetryableRefreshListenerTests.java
index a0641c365a2a1..e0ad09efac367 100644
--- a/server/src/test/java/org/opensearch/index/shard/ReleasableRetryableRefreshListenerTests.java
+++ b/server/src/test/java/org/opensearch/index/shard/ReleasableRetryableRefreshListenerTests.java
@@ -316,7 +316,32 @@ protected Logger getLogger() {
     public void testScheduleRetryAfterClose() throws Exception {
         // This tests that once the listener has been closed, even the retries would not be scheduled.
         final AtomicLong runCount = new AtomicLong();
-        ReleasableRetryableRefreshListener testRefreshListener = new ReleasableRetryableRefreshListener(threadPool) {
+        ReleasableRetryableRefreshListener testRefreshListener = getRetryableRefreshListener(runCount);
+        Thread thread1 = new Thread(() -> {
+            try {
+                testRefreshListener.afterRefresh(true);
+            } catch (IOException e) {
+                throw new AssertionError(e);
+            }
+        });
+        Thread thread2 = new Thread(() -> {
+            try {
+                Thread.sleep(500);
+                testRefreshListener.drainRefreshes();
+            } catch (InterruptedException e) {
+                throw new AssertionError(e);
+            }
+        });
+        thread1.start();
+        thread2.start();
+        thread1.join();
+        thread2.join();
+        assertBusy(() -> assertEquals(1, runCount.get()));
+        assertRefreshListenerClosed(testRefreshListener);
+    }
+
+    private ReleasableRetryableRefreshListener getRetryableRefreshListener(AtomicLong runCount) {
+        return new ReleasableRetryableRefreshListener(threadPool) {
             @Override
             protected boolean performAfterRefreshWithPermit(boolean didRefresh) {
                 try {
@@ -341,6 +366,11 @@ protected String getRetryThreadPoolName() {
                 return ThreadPool.Names.REMOTE_REFRESH_RETRY;
             }
 
+            @Override
+            protected boolean isRetryEnabled() {
+                return true;
+            }
+
             @Override
             protected TimeValue getNextRetryInterval() {
                 try {
@@ -351,6 +381,12 @@ protected TimeValue getNextRetryInterval() {
                 return TimeValue.timeValueMillis(100);
             }
         };
+    }
+
+    public void testScheduleRetryAfterThreadpoolShutdown() throws Exception {
+        // This tests that once the thread-pool is shut down, the exception is handled.
+        final AtomicLong runCount = new AtomicLong();
+        ReleasableRetryableRefreshListener testRefreshListener = getRetryableRefreshListener(runCount);
         Thread thread1 = new Thread(() -> {
             try {
                 testRefreshListener.afterRefresh(true);
@@ -361,7 +397,7 @@ protected TimeValue getNextRetryInterval() {
         Thread thread2 = new Thread(() -> {
             try {
                 Thread.sleep(500);
-                testRefreshListener.drainRefreshes();
+                threadPool.shutdown();
             } catch (InterruptedException e) {
                 throw new AssertionError(e);
             }
@@ -371,7 +407,7 @@ protected TimeValue getNextRetryInterval() {
         thread1.join();
         thread2.join();
         assertBusy(() -> assertEquals(1, runCount.get()));
-        assertRefreshListenerClosed(testRefreshListener);
+        assertFalse(testRefreshListener.getRetryScheduledStatus());
     }
 
     public void testConcurrentScheduleRetry() throws Exception {
diff --git a/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java b/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java
index ed4f6f9511add..8fbff4527ec7b 100644
--- a/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java
+++ b/server/src/test/java/org/opensearch/index/shard/RemoteStoreRefreshListenerTests.java
@@ -34,7 +34,7 @@
 import org.opensearch.index.store.RemoteSegmentStoreDirectory.MetadataFilenameUtils;
 import org.opensearch.index.store.Store;
 import org.opensearch.index.store.lockmanager.RemoteStoreLockManager;
-import org.opensearch.indices.recovery.RecoverySettings;
+import org.opensearch.indices.RemoteStoreSettings;
 import org.opensearch.indices.replication.checkpoint.SegmentReplicationCheckpointPublisher;
 import org.opensearch.indices.replication.common.ReplicationType;
 import org.opensearch.test.ClusterServiceUtils;
@@ -250,7 +250,7 @@ public void testAfterMultipleCommits() throws IOException {
         setup(true, 3);
         assertDocs(indexShard, "1", "2", "3");
 
-        for (int i = 0; i < indexShard.getRecoverySettings().getMinRemoteSegmentMetadataFiles() + 3; i++) {
+        for (int i = 0; i < indexShard.getRemoteStoreSettings().getMinRemoteSegmentMetadataFiles() + 3; i++) {
             indexDocs(4 * (i + 1), 4);
             flushShard(indexShard);
         }
@@ -636,9 +636,9 @@ private Tuple<RemoteStoreRefreshListener, RemoteStoreStatsTrackerFactory> mockIn
         RemoteStoreStatsTrackerFactory remoteStoreStatsTrackerFactory = indexShard.getRemoteStoreStatsTrackerFactory();
         when(shard.indexSettings()).thenReturn(indexShard.indexSettings());
         when(shard.shardId()).thenReturn(indexShard.shardId());
-        RecoverySettings recoverySettings = mock(RecoverySettings.class);
-        when(recoverySettings.getMinRemoteSegmentMetadataFiles()).thenReturn(10);
-        when(shard.getRecoverySettings()).thenReturn(recoverySettings);
+        RemoteStoreSettings remoteStoreSettings = mock(RemoteStoreSettings.class);
+        when(remoteStoreSettings.getMinRemoteSegmentMetadataFiles()).thenReturn(10);
+        when(shard.getRemoteStoreSettings()).thenReturn(remoteStoreSettings);
         RemoteStoreRefreshListener refreshListener = new RemoteStoreRefreshListener(shard, emptyCheckpointPublisher, tracker);
         refreshListener.afterRefresh(true);
         return Tuple.tuple(refreshListener, remoteStoreStatsTrackerFactory);
diff --git a/server/src/test/java/org/opensearch/index/shard/SearchOperationListenerTests.java b/server/src/test/java/org/opensearch/index/shard/SearchOperationListenerTests.java
index 98f86758ea2ca..c61c13eecf2c3 100644
--- a/server/src/test/java/org/opensearch/index/shard/SearchOperationListenerTests.java
+++ b/server/src/test/java/org/opensearch/index/shard/SearchOperationListenerTests.java
@@ -64,6 +64,7 @@ public void testListenersAreExecuted() {
         AtomicInteger newScrollContext = new AtomicInteger();
         AtomicInteger freeScrollContext = new AtomicInteger();
         AtomicInteger validateSearchContext = new AtomicInteger();
+        AtomicInteger searchIdleReactivateCount = new AtomicInteger();
         AtomicInteger timeInNanos = new AtomicInteger(randomIntBetween(0, 10));
         SearchOperationListener listener = new SearchOperationListener() {
             @Override
@@ -133,6 +134,11 @@ public void validateReaderContext(ReaderContext readerContext, TransportRequest
                 assertNotNull(readerContext);
                 validateSearchContext.incrementAndGet();
             }
+
+            @Override
+            public void onSearchIdleReactivation() {
+                searchIdleReactivateCount.incrementAndGet();
+            }
         };
 
         SearchOperationListener throwingListener = (SearchOperationListener) Proxy.newProxyInstance(
@@ -169,6 +175,7 @@ public void validateReaderContext(ReaderContext readerContext, TransportRequest
         assertEquals(0, newScrollContext.get());
         assertEquals(0, freeContext.get());
         assertEquals(0, freeScrollContext.get());
+        assertEquals(0, searchIdleReactivateCount.get());
         assertEquals(0, validateSearchContext.get());
 
         compositeListener.onFetchPhase(ctx, timeInNanos.get());
@@ -182,6 +189,7 @@ public void validateReaderContext(ReaderContext readerContext, TransportRequest
         assertEquals(0, newScrollContext.get());
         assertEquals(0, freeContext.get());
         assertEquals(0, freeScrollContext.get());
+        assertEquals(0, searchIdleReactivateCount.get());
         assertEquals(0, validateSearchContext.get());
 
         compositeListener.onPreQueryPhase(ctx);
@@ -195,6 +203,7 @@ public void validateReaderContext(ReaderContext readerContext, TransportRequest
         assertEquals(0, newScrollContext.get());
         assertEquals(0, freeContext.get());
         assertEquals(0, freeScrollContext.get());
+        assertEquals(0, searchIdleReactivateCount.get());
         assertEquals(0, validateSearchContext.get());
 
         compositeListener.onPreFetchPhase(ctx);
@@ -208,6 +217,7 @@ public void validateReaderContext(ReaderContext readerContext, TransportRequest
         assertEquals(0, newScrollContext.get());
         assertEquals(0, freeContext.get());
         assertEquals(0, freeScrollContext.get());
+        assertEquals(0, searchIdleReactivateCount.get());
         assertEquals(0, validateSearchContext.get());
 
         compositeListener.onFailedFetchPhase(ctx);
@@ -221,6 +231,7 @@ public void validateReaderContext(ReaderContext readerContext, TransportRequest
         assertEquals(0, newScrollContext.get());
         assertEquals(0, freeContext.get());
         assertEquals(0, freeScrollContext.get());
+        assertEquals(0, searchIdleReactivateCount.get());
         assertEquals(0, validateSearchContext.get());
 
         compositeListener.onFailedQueryPhase(ctx);
@@ -234,6 +245,7 @@ public void validateReaderContext(ReaderContext readerContext, TransportRequest
         assertEquals(0, newScrollContext.get());
         assertEquals(0, freeContext.get());
         assertEquals(0, freeScrollContext.get());
+        assertEquals(0, searchIdleReactivateCount.get());
         assertEquals(0, validateSearchContext.get());
 
         compositeListener.onNewReaderContext(mock(ReaderContext.class));
@@ -247,6 +259,7 @@ public void validateReaderContext(ReaderContext readerContext, TransportRequest
         assertEquals(0, newScrollContext.get());
         assertEquals(0, freeContext.get());
         assertEquals(0, freeScrollContext.get());
+        assertEquals(0, searchIdleReactivateCount.get());
         assertEquals(0, validateSearchContext.get());
 
         compositeListener.onNewScrollContext(mock(ReaderContext.class));
@@ -260,6 +273,7 @@ public void validateReaderContext(ReaderContext readerContext, TransportRequest
         assertEquals(2, newScrollContext.get());
         assertEquals(0, freeContext.get());
         assertEquals(0, freeScrollContext.get());
+        assertEquals(0, searchIdleReactivateCount.get());
         assertEquals(0, validateSearchContext.get());
 
         compositeListener.onFreeReaderContext(mock(ReaderContext.class));
@@ -273,6 +287,7 @@ public void validateReaderContext(ReaderContext readerContext, TransportRequest
         assertEquals(2, newScrollContext.get());
         assertEquals(2, freeContext.get());
         assertEquals(0, freeScrollContext.get());
+        assertEquals(0, searchIdleReactivateCount.get());
         assertEquals(0, validateSearchContext.get());
 
         compositeListener.onFreeScrollContext(mock(ReaderContext.class));
@@ -286,6 +301,21 @@ public void validateReaderContext(ReaderContext readerContext, TransportRequest
         assertEquals(2, newScrollContext.get());
         assertEquals(2, freeContext.get());
         assertEquals(2, freeScrollContext.get());
+        assertEquals(0, searchIdleReactivateCount.get());
+        assertEquals(0, validateSearchContext.get());
+
+        compositeListener.onSearchIdleReactivation();
+        assertEquals(2, preFetch.get());
+        assertEquals(2, preQuery.get());
+        assertEquals(2, failedFetch.get());
+        assertEquals(2, failedQuery.get());
+        assertEquals(2, onQuery.get());
+        assertEquals(2, onFetch.get());
+        assertEquals(2, newContext.get());
+        assertEquals(2, newScrollContext.get());
+        assertEquals(2, freeContext.get());
+        assertEquals(2, freeScrollContext.get());
+        assertEquals(2, searchIdleReactivateCount.get());
         assertEquals(0, validateSearchContext.get());
 
         if (throwingListeners == 0) {
@@ -311,6 +341,7 @@ public void validateReaderContext(ReaderContext readerContext, TransportRequest
         assertEquals(2, newScrollContext.get());
         assertEquals(2, freeContext.get());
         assertEquals(2, freeScrollContext.get());
+        assertEquals(2, searchIdleReactivateCount.get());
         assertEquals(2, validateSearchContext.get());
     }
 }
diff --git a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java
index e93d266dcab4c..2311fc582616f 100644
--- a/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java
+++ b/server/src/test/java/org/opensearch/index/shard/SegmentReplicationIndexShardTests.java
@@ -1019,25 +1019,15 @@ protected void assertDocCounts(IndexShard indexShard, int expectedPersistedDocCo
     }
 
     protected void resolveCheckpointInfoResponseListener(ActionListener<CheckpointInfoResponse> listener, IndexShard primary) {
-        final CopyState copyState;
-        try {
-            copyState = new CopyState(
-                ReplicationCheckpoint.empty(primary.shardId, primary.getLatestReplicationCheckpoint().getCodec()),
-                primary
+        try (final CopyState copyState = new CopyState(primary)) {
+            listener.onResponse(
+                new CheckpointInfoResponse(copyState.getCheckpoint(), copyState.getMetadataMap(), copyState.getInfosBytes())
             );
         } catch (IOException e) {
             logger.error("Unexpected error computing CopyState", e);
             Assert.fail("Failed to compute copyState");
             throw new UncheckedIOException(e);
         }
-
-        try {
-            listener.onResponse(
-                new CheckpointInfoResponse(copyState.getCheckpoint(), copyState.getMetadataMap(), copyState.getInfosBytes())
-            );
-        } finally {
-            copyState.decRef();
-        }
     }
 
     protected void startReplicationAndAssertCancellation(
diff --git a/server/src/test/java/org/opensearch/index/snapshots/blobstore/RemoteStoreShardShallowCopySnapshotTests.java b/server/src/test/java/org/opensearch/index/snapshots/blobstore/RemoteStoreShardShallowCopySnapshotTests.java
index 38c4bb781ce06..e81eef67d6704 100644
--- a/server/src/test/java/org/opensearch/index/snapshots/blobstore/RemoteStoreShardShallowCopySnapshotTests.java
+++ b/server/src/test/java/org/opensearch/index/snapshots/blobstore/RemoteStoreShardShallowCopySnapshotTests.java
@@ -14,6 +14,8 @@
 import org.opensearch.core.xcontent.ToXContent;
 import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.core.xcontent.XContentParser;
+import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm;
+import org.opensearch.index.remote.RemoteStoreEnums.PathType;
 import org.opensearch.test.OpenSearchTestCase;
 
 import java.io.IOException;
@@ -40,7 +42,10 @@ public void testToXContent() throws IOException {
         String repositoryBasePath = "test-repo-basepath";
         List<String> fileNames = new ArrayList<>(5);
         fileNames.addAll(Arrays.asList("file1", "file2", "file3", "file4", "file5"));
+
+        // Case 1 - Without remote path type fields
         RemoteStoreShardShallowCopySnapshot shardShallowCopySnapshot = new RemoteStoreShardShallowCopySnapshot(
+            "1",
             snapshot,
             indexVersion,
             primaryTerm,
@@ -52,7 +57,9 @@ public void testToXContent() throws IOException {
             indexUUID,
             remoteStoreRepository,
             repositoryBasePath,
-            fileNames
+            fileNames,
+            null,
+            null
         );
         String actual;
         try (XContentBuilder builder = MediaTypeRegistry.JSON.contentBuilder()) {
@@ -66,6 +73,160 @@ public void testToXContent() throws IOException {
             + "\"test-rs-repository\",\"commit_generation\":5,\"primary_term\":3,\"remote_store_repository_base_path\":"
             + "\"test-repo-basepath\",\"file_names\":[\"file1\",\"file2\",\"file3\",\"file4\",\"file5\"]}";
         assert Objects.equals(actual, expectedXContent) : "xContent is " + actual;
+
+        // Case 2 - with just fixed type
+        shardShallowCopySnapshot = new RemoteStoreShardShallowCopySnapshot(
+            snapshot,
+            indexVersion,
+            primaryTerm,
+            commitGeneration,
+            startTime,
+            time,
+            totalFileCount,
+            totalSize,
+            indexUUID,
+            remoteStoreRepository,
+            repositoryBasePath,
+            fileNames,
+            PathType.FIXED,
+            null
+        );
+        try (XContentBuilder builder = MediaTypeRegistry.JSON.contentBuilder()) {
+            builder.startObject();
+            shardShallowCopySnapshot.toXContent(builder, ToXContent.EMPTY_PARAMS);
+            builder.endObject();
+            actual = builder.toString();
+        }
+
+        expectedXContent = "{\"version\":\"2\",\"name\":\"test-snapshot\",\"index_version\":1,\"start_time\":123,\"time\":123,"
+            + "\"number_of_files\":5,\"total_size\":5,\"index_uuid\":\"syzhajds-ashdlfj\",\"remote_store_repository\":"
+            + "\"test-rs-repository\",\"commit_generation\":5,\"primary_term\":3,\"remote_store_repository_base_path\":"
+            + "\"test-repo-basepath\",\"file_names\":[\"file1\",\"file2\",\"file3\",\"file4\",\"file5\"],\"path_type\":0}";
+        assert Objects.equals(actual, expectedXContent) : "xContent is " + actual;
+
+        // Case 3 - with just hashed prefix type and FNV_1A_BASE64 hash algorithm
+        shardShallowCopySnapshot = new RemoteStoreShardShallowCopySnapshot(
+            snapshot,
+            indexVersion,
+            primaryTerm,
+            commitGeneration,
+            startTime,
+            time,
+            totalFileCount,
+            totalSize,
+            indexUUID,
+            remoteStoreRepository,
+            repositoryBasePath,
+            fileNames,
+            PathType.HASHED_PREFIX,
+            PathHashAlgorithm.FNV_1A_BASE64
+        );
+        try (XContentBuilder builder = MediaTypeRegistry.JSON.contentBuilder()) {
+            builder.startObject();
+            shardShallowCopySnapshot.toXContent(builder, ToXContent.EMPTY_PARAMS);
+            builder.endObject();
+            actual = builder.toString();
+        }
+
+        expectedXContent = "{\"version\":\"2\",\"name\":\"test-snapshot\",\"index_version\":1,\"start_time\":123,\"time\":123,"
+            + "\"number_of_files\":5,\"total_size\":5,\"index_uuid\":\"syzhajds-ashdlfj\",\"remote_store_repository\":"
+            + "\"test-rs-repository\",\"commit_generation\":5,\"primary_term\":3,\"remote_store_repository_base_path\":"
+            + "\"test-repo-basepath\",\"file_names\":[\"file1\",\"file2\",\"file3\",\"file4\",\"file5\"],\"path_type\":1"
+            + ",\"path_hash_algorithm\":0}";
+        assert Objects.equals(actual, expectedXContent) : "xContent is " + actual;
+
+        // Case 4 - with just hashed prefix type and FNV_1A_COMPOSITE hash algorithm
+        shardShallowCopySnapshot = new RemoteStoreShardShallowCopySnapshot(
+            snapshot,
+            indexVersion,
+            primaryTerm,
+            commitGeneration,
+            startTime,
+            time,
+            totalFileCount,
+            totalSize,
+            indexUUID,
+            remoteStoreRepository,
+            repositoryBasePath,
+            fileNames,
+            PathType.HASHED_PREFIX,
+            PathHashAlgorithm.FNV_1A_COMPOSITE_1
+        );
+        try (XContentBuilder builder = MediaTypeRegistry.JSON.contentBuilder()) {
+            builder.startObject();
+            shardShallowCopySnapshot.toXContent(builder, ToXContent.EMPTY_PARAMS);
+            builder.endObject();
+            actual = builder.toString();
+        }
+
+        expectedXContent = "{\"version\":\"2\",\"name\":\"test-snapshot\",\"index_version\":1,\"start_time\":123,\"time\":123,"
+            + "\"number_of_files\":5,\"total_size\":5,\"index_uuid\":\"syzhajds-ashdlfj\",\"remote_store_repository\":"
+            + "\"test-rs-repository\",\"commit_generation\":5,\"primary_term\":3,\"remote_store_repository_base_path\":"
+            + "\"test-repo-basepath\",\"file_names\":[\"file1\",\"file2\",\"file3\",\"file4\",\"file5\"],\"path_type\":1"
+            + ",\"path_hash_algorithm\":1}";
+        assert Objects.equals(actual, expectedXContent) : "xContent is " + actual;
+
+        // Case 5 - with just hashed infix type and FNV_1A_BASE64 hash algorithm
+        shardShallowCopySnapshot = new RemoteStoreShardShallowCopySnapshot(
+            snapshot,
+            indexVersion,
+            primaryTerm,
+            commitGeneration,
+            startTime,
+            time,
+            totalFileCount,
+            totalSize,
+            indexUUID,
+            remoteStoreRepository,
+            repositoryBasePath,
+            fileNames,
+            PathType.HASHED_INFIX,
+            PathHashAlgorithm.FNV_1A_BASE64
+        );
+        try (XContentBuilder builder = MediaTypeRegistry.JSON.contentBuilder()) {
+            builder.startObject();
+            shardShallowCopySnapshot.toXContent(builder, ToXContent.EMPTY_PARAMS);
+            builder.endObject();
+            actual = builder.toString();
+        }
+
+        expectedXContent = "{\"version\":\"2\",\"name\":\"test-snapshot\",\"index_version\":1,\"start_time\":123,\"time\":123,"
+            + "\"number_of_files\":5,\"total_size\":5,\"index_uuid\":\"syzhajds-ashdlfj\",\"remote_store_repository\":"
+            + "\"test-rs-repository\",\"commit_generation\":5,\"primary_term\":3,\"remote_store_repository_base_path\":"
+            + "\"test-repo-basepath\",\"file_names\":[\"file1\",\"file2\",\"file3\",\"file4\",\"file5\"],\"path_type\":2"
+            + ",\"path_hash_algorithm\":0}";
+        assert Objects.equals(actual, expectedXContent) : "xContent is " + actual;
+
+        // Case 6 - with just hashed infix type and FNV_1A_COMPOSITE hash algorithm
+        shardShallowCopySnapshot = new RemoteStoreShardShallowCopySnapshot(
+            snapshot,
+            indexVersion,
+            primaryTerm,
+            commitGeneration,
+            startTime,
+            time,
+            totalFileCount,
+            totalSize,
+            indexUUID,
+            remoteStoreRepository,
+            repositoryBasePath,
+            fileNames,
+            PathType.HASHED_INFIX,
+            PathHashAlgorithm.FNV_1A_COMPOSITE_1
+        );
+        try (XContentBuilder builder = MediaTypeRegistry.JSON.contentBuilder()) {
+            builder.startObject();
+            shardShallowCopySnapshot.toXContent(builder, ToXContent.EMPTY_PARAMS);
+            builder.endObject();
+            actual = builder.toString();
+        }
+
+        expectedXContent = "{\"version\":\"2\",\"name\":\"test-snapshot\",\"index_version\":1,\"start_time\":123,\"time\":123,"
+            + "\"number_of_files\":5,\"total_size\":5,\"index_uuid\":\"syzhajds-ashdlfj\",\"remote_store_repository\":"
+            + "\"test-rs-repository\",\"commit_generation\":5,\"primary_term\":3,\"remote_store_repository_base_path\":"
+            + "\"test-repo-basepath\",\"file_names\":[\"file1\",\"file2\",\"file3\",\"file4\",\"file5\"],\"path_type\":2"
+            + ",\"path_hash_algorithm\":1}";
+        assert Objects.equals(actual, expectedXContent) : "xContent is " + actual;
     }
 
     public void testFromXContent() throws IOException {
@@ -83,6 +244,7 @@ public void testFromXContent() throws IOException {
         List<String> fileNames = new ArrayList<>(5);
         fileNames.addAll(Arrays.asList("file1", "file2", "file3", "file4", "file5"));
         RemoteStoreShardShallowCopySnapshot expectedShardShallowCopySnapshot = new RemoteStoreShardShallowCopySnapshot(
+            "1",
             snapshot,
             indexVersion,
             primaryTerm,
@@ -94,7 +256,9 @@ public void testFromXContent() throws IOException {
             indexUUID,
             remoteStoreRepository,
             repositoryBasePath,
-            fileNames
+            fileNames,
+            null,
+            null
         );
         String xContent = "{\"version\":\"1\",\"name\":\"test-snapshot\",\"index_version\":1,\"start_time\":123,\"time\":123,"
             + "\"number_of_files\":5,\"total_size\":5,\"index_uuid\":\"syzhajds-ashdlfj\",\"remote_store_repository\":"
@@ -102,22 +266,147 @@ public void testFromXContent() throws IOException {
             + "\"test-repo-basepath\",\"file_names\":[\"file1\",\"file2\",\"file3\",\"file4\",\"file5\"]}";
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, xContent)) {
             RemoteStoreShardShallowCopySnapshot actualShardShallowCopySnapshot = RemoteStoreShardShallowCopySnapshot.fromXContent(parser);
-            assertEquals(actualShardShallowCopySnapshot.snapshot(), expectedShardShallowCopySnapshot.snapshot());
-            assertEquals(
-                actualShardShallowCopySnapshot.getRemoteStoreRepository(),
-                expectedShardShallowCopySnapshot.getRemoteStoreRepository()
-            );
-            assertEquals(actualShardShallowCopySnapshot.getCommitGeneration(), expectedShardShallowCopySnapshot.getCommitGeneration());
-            assertEquals(actualShardShallowCopySnapshot.getPrimaryTerm(), expectedShardShallowCopySnapshot.getPrimaryTerm());
-            assertEquals(actualShardShallowCopySnapshot.startTime(), expectedShardShallowCopySnapshot.startTime());
-            assertEquals(actualShardShallowCopySnapshot.time(), expectedShardShallowCopySnapshot.time());
-            assertEquals(actualShardShallowCopySnapshot.totalSize(), expectedShardShallowCopySnapshot.totalSize());
-            assertEquals(actualShardShallowCopySnapshot.totalFileCount(), expectedShardShallowCopySnapshot.totalFileCount());
+            assert Objects.equals(expectedShardShallowCopySnapshot, actualShardShallowCopySnapshot);
+        }
+
+        // with pathType=PathType.FIXED
+        xContent = "{\"version\":\"2\",\"name\":\"test-snapshot\",\"index_version\":1,\"start_time\":123,\"time\":123,"
+            + "\"number_of_files\":5,\"total_size\":5,\"index_uuid\":\"syzhajds-ashdlfj\",\"remote_store_repository\":"
+            + "\"test-rs-repository\",\"commit_generation\":5,\"primary_term\":3,\"remote_store_repository_base_path\":"
+            + "\"test-repo-basepath\",\"file_names\":[\"file1\",\"file2\",\"file3\",\"file4\",\"file5\"],\"path_type\":0}";
+        expectedShardShallowCopySnapshot = new RemoteStoreShardShallowCopySnapshot(
+            "2",
+            snapshot,
+            indexVersion,
+            primaryTerm,
+            commitGeneration,
+            startTime,
+            time,
+            totalFileCount,
+            totalSize,
+            indexUUID,
+            remoteStoreRepository,
+            repositoryBasePath,
+            fileNames,
+            PathType.FIXED,
+            null
+        );
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, xContent)) {
+            RemoteStoreShardShallowCopySnapshot actualShardShallowCopySnapshot = RemoteStoreShardShallowCopySnapshot.fromXContent(parser);
+            assert Objects.equals(expectedShardShallowCopySnapshot, actualShardShallowCopySnapshot);
+        }
+
+        // with pathType=PathType.HASHED_PREFIX and pathHashAlgorithm=PathHashAlgorithm.FNV_1A
+        xContent = "{\"version\":\"2\",\"name\":\"test-snapshot\",\"index_version\":1,\"start_time\":123,\"time\":123,"
+            + "\"number_of_files\":5,\"total_size\":5,\"index_uuid\":\"syzhajds-ashdlfj\",\"remote_store_repository\":"
+            + "\"test-rs-repository\",\"commit_generation\":5,\"primary_term\":3,\"remote_store_repository_base_path\":"
+            + "\"test-repo-basepath\",\"file_names\":[\"file1\",\"file2\",\"file3\",\"file4\",\"file5\"],\"path_type\":1,\"path_hash_algorithm\":0}";
+        expectedShardShallowCopySnapshot = new RemoteStoreShardShallowCopySnapshot(
+            "2",
+            snapshot,
+            indexVersion,
+            primaryTerm,
+            commitGeneration,
+            startTime,
+            time,
+            totalFileCount,
+            totalSize,
+            indexUUID,
+            remoteStoreRepository,
+            repositoryBasePath,
+            fileNames,
+            PathType.HASHED_PREFIX,
+            PathHashAlgorithm.FNV_1A_BASE64
+        );
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, xContent)) {
+            RemoteStoreShardShallowCopySnapshot actualShardShallowCopySnapshot = RemoteStoreShardShallowCopySnapshot.fromXContent(parser);
+            assert Objects.equals(expectedShardShallowCopySnapshot, actualShardShallowCopySnapshot);
+        }
+
+        // with pathType=PathType.HASHED_PREFIX and pathHashAlgorithm=PathHashAlgorithm.FNV_1A_COMPOSITE
+        xContent = "{\"version\":\"2\",\"name\":\"test-snapshot\",\"index_version\":1,\"start_time\":123,\"time\":123,"
+            + "\"number_of_files\":5,\"total_size\":5,\"index_uuid\":\"syzhajds-ashdlfj\",\"remote_store_repository\":"
+            + "\"test-rs-repository\",\"commit_generation\":5,\"primary_term\":3,\"remote_store_repository_base_path\":"
+            + "\"test-repo-basepath\",\"file_names\":[\"file1\",\"file2\",\"file3\",\"file4\",\"file5\"],\"path_type\":1,\"path_hash_algorithm\":1}";
+        expectedShardShallowCopySnapshot = new RemoteStoreShardShallowCopySnapshot(
+            "2",
+            snapshot,
+            indexVersion,
+            primaryTerm,
+            commitGeneration,
+            startTime,
+            time,
+            totalFileCount,
+            totalSize,
+            indexUUID,
+            remoteStoreRepository,
+            repositoryBasePath,
+            fileNames,
+            PathType.HASHED_PREFIX,
+            PathHashAlgorithm.FNV_1A_COMPOSITE_1
+        );
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, xContent)) {
+            RemoteStoreShardShallowCopySnapshot actualShardShallowCopySnapshot = RemoteStoreShardShallowCopySnapshot.fromXContent(parser);
+            assert Objects.equals(expectedShardShallowCopySnapshot, actualShardShallowCopySnapshot);
+        }
+
+        // with pathType=PathType.HASHED_INFIX and pathHashAlgorithm=PathHashAlgorithm.FNV_1A
+        xContent = "{\"version\":\"2\",\"name\":\"test-snapshot\",\"index_version\":1,\"start_time\":123,\"time\":123,"
+            + "\"number_of_files\":5,\"total_size\":5,\"index_uuid\":\"syzhajds-ashdlfj\",\"remote_store_repository\":"
+            + "\"test-rs-repository\",\"commit_generation\":5,\"primary_term\":3,\"remote_store_repository_base_path\":"
+            + "\"test-repo-basepath\",\"file_names\":[\"file1\",\"file2\",\"file3\",\"file4\",\"file5\"],\"path_type\":2,\"path_hash_algorithm\":0}";
+        expectedShardShallowCopySnapshot = new RemoteStoreShardShallowCopySnapshot(
+            "2",
+            snapshot,
+            indexVersion,
+            primaryTerm,
+            commitGeneration,
+            startTime,
+            time,
+            totalFileCount,
+            totalSize,
+            indexUUID,
+            remoteStoreRepository,
+            repositoryBasePath,
+            fileNames,
+            PathType.HASHED_INFIX,
+            PathHashAlgorithm.FNV_1A_BASE64
+        );
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, xContent)) {
+            RemoteStoreShardShallowCopySnapshot actualShardShallowCopySnapshot = RemoteStoreShardShallowCopySnapshot.fromXContent(parser);
+            assert Objects.equals(expectedShardShallowCopySnapshot, actualShardShallowCopySnapshot);
+        }
+
+        // with pathType=PathType.HASHED_INFIX and pathHashAlgorithm=PathHashAlgorithm.FNV_1A_COMPOSITE
+        xContent = "{\"version\":\"2\",\"name\":\"test-snapshot\",\"index_version\":1,\"start_time\":123,\"time\":123,"
+            + "\"number_of_files\":5,\"total_size\":5,\"index_uuid\":\"syzhajds-ashdlfj\",\"remote_store_repository\":"
+            + "\"test-rs-repository\",\"commit_generation\":5,\"primary_term\":3,\"remote_store_repository_base_path\":"
+            + "\"test-repo-basepath\",\"file_names\":[\"file1\",\"file2\",\"file3\",\"file4\",\"file5\"],\"path_type\":2,\"path_hash_algorithm\":1}";
+        expectedShardShallowCopySnapshot = new RemoteStoreShardShallowCopySnapshot(
+            "2",
+            snapshot,
+            indexVersion,
+            primaryTerm,
+            commitGeneration,
+            startTime,
+            time,
+            totalFileCount,
+            totalSize,
+            indexUUID,
+            remoteStoreRepository,
+            repositoryBasePath,
+            fileNames,
+            PathType.HASHED_INFIX,
+            PathHashAlgorithm.FNV_1A_COMPOSITE_1
+        );
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, xContent)) {
+            RemoteStoreShardShallowCopySnapshot actualShardShallowCopySnapshot = RemoteStoreShardShallowCopySnapshot.fromXContent(parser);
+            assert Objects.equals(expectedShardShallowCopySnapshot, actualShardShallowCopySnapshot);
         }
     }
 
     public void testFromXContentInvalid() throws IOException {
-        final int iters = scaledRandomIntBetween(1, 10);
+        final int iters = 18;
         for (int iter = 0; iter < iters; iter++) {
             String snapshot = "test-snapshot";
             long indexVersion = 1;
@@ -133,10 +422,11 @@ public void testFromXContentInvalid() throws IOException {
             List<String> fileNames = new ArrayList<>(5);
             fileNames.addAll(Arrays.asList("file1", "file2", "file3", "file4", "file5"));
             String failure = null;
-            String version = RemoteStoreShardShallowCopySnapshot.DEFAULT_VERSION;
-            long length = Math.max(0, Math.abs(randomLong()));
+            String version = "1";
+            PathType pathType = null;
+            PathHashAlgorithm pathHashAlgorithm = null;
             // random corruption
-            switch (randomIntBetween(0, 8)) {
+            switch (iter) {
                 case 0:
                     snapshot = null;
                     failure = "Invalid/Missing Snapshot Name";
@@ -170,6 +460,57 @@ public void testFromXContentInvalid() throws IOException {
                     failure = "Invalid Version Provided";
                     break;
                 case 8:
+                    version = "2";
+                    failure = "Invalid combination of pathType=null pathHashAlgorithm=null for version=2";
+                    break;
+                case 9:
+                    version = "1";
+                    pathType = PathType.FIXED;
+                    failure = "Invalid combination of pathType=FIXED pathHashAlgorithm=null for version=1";
+                    break;
+                case 10:
+                    version = "1";
+                    pathHashAlgorithm = PathHashAlgorithm.FNV_1A_BASE64;
+                    failure = "Invalid combination of pathType=null pathHashAlgorithm=FNV_1A_BASE64 for version=1";
+                    break;
+                case 11:
+                    version = "2";
+                    pathType = PathType.FIXED;
+                    pathHashAlgorithm = PathHashAlgorithm.FNV_1A_BASE64;
+                    failure = "Invalid combination of pathType=FIXED pathHashAlgorithm=FNV_1A_BASE64 for version=2";
+                    break;
+                case 12:
+                    version = "1";
+                    pathHashAlgorithm = PathHashAlgorithm.FNV_1A_COMPOSITE_1;
+                    failure = "Invalid combination of pathType=null pathHashAlgorithm=FNV_1A_COMPOSITE_1 for version=1";
+                    break;
+                case 13:
+                    version = "2";
+                    pathType = PathType.FIXED;
+                    pathHashAlgorithm = PathHashAlgorithm.FNV_1A_COMPOSITE_1;
+                    failure = "Invalid combination of pathType=FIXED pathHashAlgorithm=FNV_1A_COMPOSITE_1 for version=2";
+                    break;
+                case 14:
+                    version = "2";
+                    pathType = PathType.HASHED_PREFIX;
+                    pathHashAlgorithm = PathHashAlgorithm.FNV_1A_BASE64;
+                    break;
+                case 15:
+                    version = "2";
+                    pathType = PathType.HASHED_PREFIX;
+                    pathHashAlgorithm = PathHashAlgorithm.FNV_1A_COMPOSITE_1;
+                    break;
+                case 16:
+                    version = "2";
+                    pathType = PathType.HASHED_INFIX;
+                    pathHashAlgorithm = PathHashAlgorithm.FNV_1A_BASE64;
+                    break;
+                case 17:
+                    version = "2";
+                    pathType = PathType.HASHED_INFIX;
+                    pathHashAlgorithm = PathHashAlgorithm.FNV_1A_COMPOSITE_1;
+                    break;
+                case 18:
                     break;
                 default:
                     fail("shouldn't be here");
@@ -194,6 +535,14 @@ public void testFromXContentInvalid() throws IOException {
                 builder.value(fileName);
             }
             builder.endArray();
+            // We are handling NP check since a cluster can have indexes created earlier which do not have remote store
+            // path type and path hash algorithm in its custom data in index metadata.
+            if (Objects.nonNull(pathType)) {
+                builder.field(RemoteStoreShardShallowCopySnapshot.PATH_TYPE, pathType.getCode());
+            }
+            if (Objects.nonNull(pathHashAlgorithm)) {
+                builder.field(RemoteStoreShardShallowCopySnapshot.PATH_HASH_ALGORITHM, pathHashAlgorithm.getCode());
+            }
             builder.endObject();
             byte[] xContent = BytesReference.toBytes(BytesReference.bytes(builder));
 
@@ -211,7 +560,8 @@ public void testFromXContentInvalid() throws IOException {
                 assertEquals(remoteStoreShardShallowCopySnapshot.startTime(), startTime);
                 assertEquals(remoteStoreShardShallowCopySnapshot.time(), time);
                 assertEquals(remoteStoreShardShallowCopySnapshot.totalSize(), totalSize);
-                assertEquals(remoteStoreShardShallowCopySnapshot.totalFileCount(), totalFileCount);
+                assertEquals(remoteStoreShardShallowCopySnapshot.getPathType(), pathType);
+                assertEquals(remoteStoreShardShallowCopySnapshot.getPathHashAlgorithm(), pathHashAlgorithm);
             } else {
                 try (XContentParser parser = createParser(JsonXContent.jsonXContent, xContent)) {
                     parser.nextToken();
diff --git a/server/src/test/java/org/opensearch/index/store/RemoteDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteDirectoryTests.java
index 9e38e1749d434..ee81369725e6f 100644
--- a/server/src/test/java/org/opensearch/index/store/RemoteDirectoryTests.java
+++ b/server/src/test/java/org/opensearch/index/store/RemoteDirectoryTests.java
@@ -104,7 +104,8 @@ public void onResponse(Void t) {
                 public void onFailure(Exception e) {
                     fail("Listener responded with exception" + e);
                 }
-            }
+            },
+            false
         );
         assertTrue(countDownLatch.await(10, TimeUnit.SECONDS));
         assertTrue(postUploadInvoked.get());
@@ -141,7 +142,8 @@ public void onResponse(Void t) {
                 public void onFailure(Exception e) {
                     countDownLatch.countDown();
                 }
-            }
+            },
+            false
         );
         assertTrue(countDownLatch.await(10, TimeUnit.SECONDS));
         assertFalse(postUploadInvoked.get());
diff --git a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java
index 8b69c15dac9d3..567199cf64cd8 100644
--- a/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java
+++ b/server/src/test/java/org/opensearch/index/store/RemoteSegmentStoreDirectoryTests.java
@@ -37,6 +37,9 @@
 import org.opensearch.core.index.Index;
 import org.opensearch.core.index.shard.ShardId;
 import org.opensearch.index.engine.NRTReplicationEngineFactory;
+import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm;
+import org.opensearch.index.remote.RemoteStoreEnums.PathType;
+import org.opensearch.index.remote.RemoteStorePathStrategy;
 import org.opensearch.index.remote.RemoteStoreUtils;
 import org.opensearch.index.shard.IndexShard;
 import org.opensearch.index.shard.IndexShardTestCase;
@@ -636,7 +639,7 @@ public void onResponse(Void unused) {
             @Override
             public void onFailure(Exception e) {}
         };
-        remoteSegmentStoreDirectory.copyFrom(storeDirectory, filename, IOContext.DEFAULT, completionListener);
+        remoteSegmentStoreDirectory.copyFrom(storeDirectory, filename, IOContext.DEFAULT, completionListener, false);
         assertTrue(latch.await(5000, TimeUnit.SECONDS));
         assertTrue(remoteSegmentStoreDirectory.getSegmentsUploadedToRemoteStore().containsKey(filename));
         storeDirectory.close();
@@ -680,7 +683,7 @@ public void onFailure(Exception e) {
                 latch.countDown();
             }
         };
-        remoteSegmentStoreDirectory.copyFrom(storeDirectory, filename, IOContext.DEFAULT, completionListener);
+        remoteSegmentStoreDirectory.copyFrom(storeDirectory, filename, IOContext.DEFAULT, completionListener, false);
         assertTrue(latch.await(5000, TimeUnit.SECONDS));
         assertFalse(remoteSegmentStoreDirectory.getSegmentsUploadedToRemoteStore().containsKey(filename));
 
@@ -697,13 +700,23 @@ public void testCleanupAsync() throws Exception {
             threadPool,
             indexShard.shardId()
         );
-        when(remoteSegmentStoreDirectoryFactory.newDirectory(any(), any(), any())).thenReturn(remoteSegmentDirectory);
+        when(remoteSegmentStoreDirectoryFactory.newDirectory(any(), any(), any(), any())).thenReturn(remoteSegmentDirectory);
         String repositoryName = "test-repository";
         String indexUUID = "test-idx-uuid";
         ShardId shardId = new ShardId(Index.UNKNOWN_INDEX_NAME, indexUUID, Integer.parseInt("0"));
+        RemoteStorePathStrategy pathStrategy = randomFrom(
+            new RemoteStorePathStrategy(PathType.FIXED),
+            new RemoteStorePathStrategy(randomFrom(PathType.HASHED_INFIX, PathType.HASHED_PREFIX), randomFrom(PathHashAlgorithm.values()))
+        );
 
-        RemoteSegmentStoreDirectory.remoteDirectoryCleanup(remoteSegmentStoreDirectoryFactory, repositoryName, indexUUID, shardId);
-        verify(remoteSegmentStoreDirectoryFactory).newDirectory(repositoryName, indexUUID, shardId);
+        RemoteSegmentStoreDirectory.remoteDirectoryCleanup(
+            remoteSegmentStoreDirectoryFactory,
+            repositoryName,
+            indexUUID,
+            shardId,
+            pathStrategy
+        );
+        verify(remoteSegmentStoreDirectoryFactory).newDirectory(repositoryName, indexUUID, shardId, pathStrategy);
         verify(threadPool, times(0)).executor(ThreadPool.Names.REMOTE_PURGE);
         verify(remoteMetadataDirectory).delete();
         verify(remoteDataDirectory).delete();
diff --git a/server/src/test/java/org/opensearch/index/store/lockmanager/RemoteStoreLockManagerFactoryTests.java b/server/src/test/java/org/opensearch/index/store/lockmanager/RemoteStoreLockManagerFactoryTests.java
index 897785849cf7b..de3dfbdaa4778 100644
--- a/server/src/test/java/org/opensearch/index/store/lockmanager/RemoteStoreLockManagerFactoryTests.java
+++ b/server/src/test/java/org/opensearch/index/store/lockmanager/RemoteStoreLockManagerFactoryTests.java
@@ -11,6 +11,8 @@
 import org.opensearch.common.blobstore.BlobContainer;
 import org.opensearch.common.blobstore.BlobPath;
 import org.opensearch.common.blobstore.BlobStore;
+import org.opensearch.index.remote.RemoteStoreEnums.PathType;
+import org.opensearch.index.remote.RemoteStorePathStrategy;
 import org.opensearch.repositories.RepositoriesService;
 import org.opensearch.repositories.blobstore.BlobStoreRepository;
 import org.opensearch.test.OpenSearchTestCase;
@@ -48,6 +50,7 @@ public void testNewLockManager() throws IOException {
         String testRepository = "testRepository";
         String testIndexUUID = "testIndexUUID";
         String testShardId = "testShardId";
+        RemoteStorePathStrategy pathStrategy = new RemoteStorePathStrategy(PathType.FIXED);
 
         BlobStoreRepository repository = mock(BlobStoreRepository.class);
         BlobStore blobStore = mock(BlobStore.class);
@@ -59,7 +62,12 @@ public void testNewLockManager() throws IOException {
 
         when(repositoriesService.repository(testRepository)).thenReturn(repository);
 
-        RemoteStoreLockManager lockManager = remoteStoreLockManagerFactory.newLockManager(testRepository, testIndexUUID, testShardId);
+        RemoteStoreLockManager lockManager = remoteStoreLockManagerFactory.newLockManager(
+            testRepository,
+            testIndexUUID,
+            testShardId,
+            pathStrategy
+        );
 
         assertTrue(lockManager != null);
         ArgumentCaptor<BlobPath> blobPathCaptor = ArgumentCaptor.forClass(BlobPath.class);
diff --git a/server/src/test/java/org/opensearch/index/translog/InternalTranslogManagerTests.java b/server/src/test/java/org/opensearch/index/translog/InternalTranslogManagerTests.java
index c098d11a3487f..c27d0367abf68 100644
--- a/server/src/test/java/org/opensearch/index/translog/InternalTranslogManagerTests.java
+++ b/server/src/test/java/org/opensearch/index/translog/InternalTranslogManagerTests.java
@@ -38,7 +38,7 @@ public void testRecoveryFromTranslog() throws IOException {
         LocalCheckpointTracker tracker = new LocalCheckpointTracker(NO_OPS_PERFORMED, NO_OPS_PERFORMED);
         try {
             translogManager = new InternalTranslogManager(
-                new TranslogConfig(shardId, primaryTranslogDir, INDEX_SETTINGS, BigArrays.NON_RECYCLING_INSTANCE, ""),
+                new TranslogConfig(shardId, primaryTranslogDir, INDEX_SETTINGS, BigArrays.NON_RECYCLING_INSTANCE, "", false),
                 primaryTerm,
                 globalCheckpoint::get,
                 createTranslogDeletionPolicy(INDEX_SETTINGS),
@@ -68,7 +68,7 @@ public void testRecoveryFromTranslog() throws IOException {
             translogManager.syncTranslog();
             translogManager.close();
             translogManager = new InternalTranslogManager(
-                new TranslogConfig(shardId, primaryTranslogDir, INDEX_SETTINGS, BigArrays.NON_RECYCLING_INSTANCE, ""),
+                new TranslogConfig(shardId, primaryTranslogDir, INDEX_SETTINGS, BigArrays.NON_RECYCLING_INSTANCE, "", false),
                 primaryTerm,
                 globalCheckpoint::get,
                 createTranslogDeletionPolicy(INDEX_SETTINGS),
@@ -117,7 +117,7 @@ public void testTranslogRollsGeneration() throws IOException {
         LocalCheckpointTracker tracker = new LocalCheckpointTracker(NO_OPS_PERFORMED, NO_OPS_PERFORMED);
         try {
             translogManager = new InternalTranslogManager(
-                new TranslogConfig(shardId, primaryTranslogDir, INDEX_SETTINGS, BigArrays.NON_RECYCLING_INSTANCE, ""),
+                new TranslogConfig(shardId, primaryTranslogDir, INDEX_SETTINGS, BigArrays.NON_RECYCLING_INSTANCE, "", false),
                 primaryTerm,
                 globalCheckpoint::get,
                 createTranslogDeletionPolicy(INDEX_SETTINGS),
@@ -147,7 +147,7 @@ public void testTranslogRollsGeneration() throws IOException {
             translogManager.syncTranslog();
             translogManager.close();
             translogManager = new InternalTranslogManager(
-                new TranslogConfig(shardId, primaryTranslogDir, INDEX_SETTINGS, BigArrays.NON_RECYCLING_INSTANCE, ""),
+                new TranslogConfig(shardId, primaryTranslogDir, INDEX_SETTINGS, BigArrays.NON_RECYCLING_INSTANCE, "", false),
                 primaryTerm,
                 globalCheckpoint::get,
                 createTranslogDeletionPolicy(INDEX_SETTINGS),
@@ -182,7 +182,7 @@ public void testTrimOperationsFromTranslog() throws IOException {
         LocalCheckpointTracker tracker = new LocalCheckpointTracker(NO_OPS_PERFORMED, NO_OPS_PERFORMED);
         try {
             translogManager = new InternalTranslogManager(
-                new TranslogConfig(shardId, primaryTranslogDir, INDEX_SETTINGS, BigArrays.NON_RECYCLING_INSTANCE, ""),
+                new TranslogConfig(shardId, primaryTranslogDir, INDEX_SETTINGS, BigArrays.NON_RECYCLING_INSTANCE, "", false),
                 primaryTerm,
                 globalCheckpoint::get,
                 createTranslogDeletionPolicy(INDEX_SETTINGS),
@@ -214,7 +214,7 @@ public void testTrimOperationsFromTranslog() throws IOException {
 
             translogManager.close();
             translogManager = new InternalTranslogManager(
-                new TranslogConfig(shardId, primaryTranslogDir, INDEX_SETTINGS, BigArrays.NON_RECYCLING_INSTANCE, ""),
+                new TranslogConfig(shardId, primaryTranslogDir, INDEX_SETTINGS, BigArrays.NON_RECYCLING_INSTANCE, "", false),
                 primaryTerm,
                 globalCheckpoint::get,
                 createTranslogDeletionPolicy(INDEX_SETTINGS),
@@ -253,7 +253,7 @@ public void testTranslogSync() throws IOException {
             ParsedDocument doc = testParsedDocument("1", null, testDocumentWithTextField(), B_1, null);
             AtomicReference<InternalTranslogManager> translogManagerAtomicReference = new AtomicReference<>();
             translogManager = new InternalTranslogManager(
-                new TranslogConfig(shardId, primaryTranslogDir, INDEX_SETTINGS, BigArrays.NON_RECYCLING_INSTANCE, ""),
+                new TranslogConfig(shardId, primaryTranslogDir, INDEX_SETTINGS, BigArrays.NON_RECYCLING_INSTANCE, "", false),
                 primaryTerm,
                 globalCheckpoint::get,
                 createTranslogDeletionPolicy(INDEX_SETTINGS),
diff --git a/server/src/test/java/org/opensearch/index/translog/LocalTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/LocalTranslogTests.java
index 4997067b75198..cae27d5b259c4 100644
--- a/server/src/test/java/org/opensearch/index/translog/LocalTranslogTests.java
+++ b/server/src/test/java/org/opensearch/index/translog/LocalTranslogTests.java
@@ -291,7 +291,7 @@ private TranslogConfig getTranslogConfig(final Path path, final Settings setting
         );
 
         final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(shardId.getIndex(), settings);
-        return new TranslogConfig(shardId, path, indexSettings, NON_RECYCLING_INSTANCE, bufferSize, "");
+        return new TranslogConfig(shardId, path, indexSettings, NON_RECYCLING_INSTANCE, bufferSize, "", false);
     }
 
     private Location addToTranslogAndList(Translog translog, List<Translog.Operation> list, Translog.Operation op) throws IOException {
@@ -1453,7 +1453,8 @@ public void testTranslogWriterCanFlushInAddOrReadCall() throws IOException {
             temp.getIndexSettings(),
             temp.getBigArrays(),
             new ByteSizeValue(1, ByteSizeUnit.KB),
-            ""
+            "",
+            false
         );
 
         final Set<Long> persistedSeqNos = new HashSet<>();
@@ -1552,7 +1553,8 @@ public void testTranslogWriterFsyncedWithLocalTranslog() throws IOException {
             temp.getIndexSettings(),
             temp.getBigArrays(),
             new ByteSizeValue(1, ByteSizeUnit.KB),
-            ""
+            "",
+            false
         );
 
         final Set<Long> persistedSeqNos = new HashSet<>();
diff --git a/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java b/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java
index 7ff4c3ecf5236..6bf35cc1eac9b 100644
--- a/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java
+++ b/server/src/test/java/org/opensearch/index/translog/RemoteFsTranslogTests.java
@@ -50,6 +50,7 @@
 import org.opensearch.index.translog.transfer.TranslogTransferManager;
 import org.opensearch.index.translog.transfer.TranslogTransferMetadata;
 import org.opensearch.index.translog.transfer.TranslogUploadFailedException;
+import org.opensearch.indices.DefaultRemoteStoreSettings;
 import org.opensearch.indices.recovery.RecoverySettings;
 import org.opensearch.indices.replication.common.ReplicationType;
 import org.opensearch.repositories.blobstore.BlobStoreRepository;
@@ -99,7 +100,7 @@
 
 import static org.opensearch.common.util.BigArrays.NON_RECYCLING_INSTANCE;
 import static org.opensearch.index.IndexSettings.INDEX_REMOTE_TRANSLOG_KEEP_EXTRA_GEN_SETTING;
-import static org.opensearch.index.translog.RemoteFsTranslog.TRANSLOG;
+import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.TRANSLOG;
 import static org.opensearch.index.translog.SnapshotMatchers.containsOperationsInAnyOrder;
 import static org.opensearch.index.translog.TranslogDeletionPolicies.createTranslogDeletionPolicy;
 import static org.hamcrest.Matchers.contains;
@@ -188,7 +189,8 @@ private RemoteFsTranslog create(Path path, BlobStoreRepository repository, Strin
             repository,
             threadPool,
             primaryMode::get,
-            new RemoteTranslogTransferTracker(shardId, 10)
+            new RemoteTranslogTransferTracker(shardId, 10),
+            DefaultRemoteStoreSettings.INSTANCE
         );
     }
 
@@ -222,7 +224,7 @@ private TranslogConfig getTranslogConfig(final Path path, final Settings setting
         // To simulate that the node is remote backed
         Settings nodeSettings = Settings.builder().put("node.attr.remote_store.translog.repository", "my-repo-1").build();
         final IndexSettings indexSettings = IndexSettingsModule.newIndexSettings(shardId.getIndex(), settings, nodeSettings);
-        return new TranslogConfig(shardId, path, indexSettings, NON_RECYCLING_INSTANCE, bufferSize, "");
+        return new TranslogConfig(shardId, path, indexSettings, NON_RECYCLING_INSTANCE, bufferSize, "", false);
     }
 
     private BlobStoreRepository createRepository() {
@@ -397,7 +399,8 @@ private TranslogConfig getConfig(int gensToKeep) {
             temp.getIndexSettings(),
             temp.getBigArrays(),
             new ByteSizeValue(1, ByteSizeUnit.KB),
-            ""
+            "",
+            false
         );
         return config;
     }
@@ -459,7 +462,8 @@ public void testExtraGenToKeep() throws Exception {
                 repository,
                 threadPool,
                 () -> Boolean.TRUE,
-                new RemoteTranslogTransferTracker(shardId, 10)
+                new RemoteTranslogTransferTracker(shardId, 10),
+                DefaultRemoteStoreSettings.INSTANCE
             ) {
                 @Override
                 ChannelFactory getChannelFactory() {
@@ -907,7 +911,7 @@ public void testDrainSync() throws Exception {
     }
 
     private BlobPath getTranslogDirectory() {
-        return repository.basePath().add(shardId.getIndex().getUUID()).add(String.valueOf(shardId.id())).add(TRANSLOG);
+        return repository.basePath().add(shardId.getIndex().getUUID()).add(String.valueOf(shardId.id())).add(TRANSLOG.getName());
     }
 
     private Long populateTranslogOps(boolean withMissingOps) throws IOException {
@@ -1508,7 +1512,8 @@ public void testTranslogWriterCanFlushInAddOrReadCall() throws IOException {
                 repository,
                 threadPool,
                 () -> Boolean.TRUE,
-                new RemoteTranslogTransferTracker(shardId, 10)
+                new RemoteTranslogTransferTracker(shardId, 10),
+                DefaultRemoteStoreSettings.INSTANCE
             ) {
                 @Override
                 ChannelFactory getChannelFactory() {
@@ -1557,7 +1562,8 @@ public void testTranslogWriterFsyncDisabledInRemoteFsTranslog() throws IOExcepti
             temp.getIndexSettings(),
             temp.getBigArrays(),
             new ByteSizeValue(1, ByteSizeUnit.KB),
-            ""
+            "",
+            false
         );
 
         final Set<Long> persistedSeqNos = new HashSet<>();
@@ -1616,7 +1622,8 @@ public void force(boolean metaData) throws IOException {
                 repository,
                 threadPool,
                 () -> Boolean.TRUE,
-                new RemoteTranslogTransferTracker(shardId, 10)
+                new RemoteTranslogTransferTracker(shardId, 10),
+                DefaultRemoteStoreSettings.INSTANCE
             ) {
                 @Override
                 ChannelFactory getChannelFactory() {
@@ -1687,7 +1694,7 @@ public void testDownloadWithRetries() throws IOException {
         // Always File not found
         when(mockTransfer.downloadTranslog(any(), any(), any())).thenThrow(new NoSuchFileException("File not found"));
         TranslogTransferManager finalMockTransfer = mockTransfer;
-        assertThrows(NoSuchFileException.class, () -> RemoteFsTranslog.download(finalMockTransfer, location, logger));
+        assertThrows(NoSuchFileException.class, () -> RemoteFsTranslog.download(finalMockTransfer, location, logger, false));
 
         // File not found in first attempt . File found in second attempt.
         mockTransfer = mock(TranslogTransferManager.class);
@@ -1708,7 +1715,83 @@ public void testDownloadWithRetries() throws IOException {
         }).when(mockTransfer).downloadTranslog(any(), any(), any());
 
         // no exception thrown
-        RemoteFsTranslog.download(mockTransfer, location, logger);
+        RemoteFsTranslog.download(mockTransfer, location, logger, false);
+    }
+
+    // No translog data in local as well as remote, we skip creating empty translog
+    public void testDownloadWithNoTranslogInLocalAndRemote() throws IOException {
+        Path location = createTempDir();
+
+        TranslogTransferManager mockTransfer = mock(TranslogTransferManager.class);
+        RemoteTranslogTransferTracker remoteTranslogTransferTracker = mock(RemoteTranslogTransferTracker.class);
+        when(mockTransfer.readMetadata()).thenReturn(null);
+        when(mockTransfer.getRemoteTranslogTransferTracker()).thenReturn(remoteTranslogTransferTracker);
+
+        Path[] filesBeforeDownload = FileSystemUtils.files(location);
+        RemoteFsTranslog.download(mockTransfer, location, logger, false);
+        assertEquals(filesBeforeDownload, FileSystemUtils.files(location));
+    }
+
+    // No translog data in remote but non-empty translog is present in local. In this case, we delete all the files
+    // from local file system and create empty translog
+    public void testDownloadWithTranslogOnlyInLocal() throws IOException {
+        TranslogTransferManager mockTransfer = mock(TranslogTransferManager.class);
+        RemoteTranslogTransferTracker remoteTranslogTransferTracker = mock(RemoteTranslogTransferTracker.class);
+        when(mockTransfer.readMetadata()).thenReturn(null);
+        when(mockTransfer.getRemoteTranslogTransferTracker()).thenReturn(remoteTranslogTransferTracker);
+
+        Path location = createTempDir();
+        for (Path file : FileSystemUtils.files(translogDir)) {
+            Files.copy(file, location.resolve(file.getFileName()));
+        }
+
+        Checkpoint existingCheckpoint = Translog.readCheckpoint(location);
+
+        TranslogTransferManager finalMockTransfer = mockTransfer;
+        RemoteFsTranslog.download(finalMockTransfer, location, logger, false);
+
+        Path[] filesPostDownload = FileSystemUtils.files(location);
+        assertEquals(2, filesPostDownload.length);
+        assertTrue(
+            filesPostDownload[0].getFileName().toString().contains("translog.ckp")
+                || filesPostDownload[1].getFileName().toString().contains("translog.ckp")
+        );
+
+        Checkpoint newEmptyTranslogCheckpoint = Translog.readCheckpoint(location);
+        // Verify that the new checkpoint points to empty translog
+        assertTrue(
+            newEmptyTranslogCheckpoint.generation == newEmptyTranslogCheckpoint.minTranslogGeneration
+                && newEmptyTranslogCheckpoint.minSeqNo == SequenceNumbers.NO_OPS_PERFORMED
+                && newEmptyTranslogCheckpoint.maxSeqNo == SequenceNumbers.NO_OPS_PERFORMED
+                && newEmptyTranslogCheckpoint.numOps == 0
+        );
+        assertTrue(newEmptyTranslogCheckpoint.generation > existingCheckpoint.generation);
+        assertEquals(newEmptyTranslogCheckpoint.globalCheckpoint, existingCheckpoint.globalCheckpoint);
+    }
+
+    // No translog data in remote and empty translog in local. We skip creating another empty translog
+    public void testDownloadWithEmptyTranslogOnlyInLocal() throws IOException {
+        TranslogTransferManager mockTransfer = mock(TranslogTransferManager.class);
+        RemoteTranslogTransferTracker remoteTranslogTransferTracker = mock(RemoteTranslogTransferTracker.class);
+        when(mockTransfer.readMetadata()).thenReturn(null);
+        when(mockTransfer.getRemoteTranslogTransferTracker()).thenReturn(remoteTranslogTransferTracker);
+
+        Path location = createTempDir();
+        for (Path file : FileSystemUtils.files(translogDir)) {
+            Files.copy(file, location.resolve(file.getFileName()));
+        }
+
+        TranslogTransferManager finalMockTransfer = mockTransfer;
+
+        // download first time will ensure creating empty translog
+        RemoteFsTranslog.download(finalMockTransfer, location, logger, false);
+        Path[] filesPostFirstDownload = FileSystemUtils.files(location);
+
+        // download on empty translog should be a no-op
+        RemoteFsTranslog.download(finalMockTransfer, location, logger, false);
+        Path[] filesPostSecondDownload = FileSystemUtils.files(location);
+
+        assertArrayEquals(filesPostFirstDownload, filesPostSecondDownload);
     }
 
     public class ThrowingBlobRepository extends FsRepository {
diff --git a/server/src/test/java/org/opensearch/index/translog/TranslogManagerTestCase.java b/server/src/test/java/org/opensearch/index/translog/TranslogManagerTestCase.java
index e17d2770f014a..a16e607bbf37a 100644
--- a/server/src/test/java/org/opensearch/index/translog/TranslogManagerTestCase.java
+++ b/server/src/test/java/org/opensearch/index/translog/TranslogManagerTestCase.java
@@ -74,7 +74,14 @@ protected Translog createTranslog(LongSupplier primaryTermSupplier) throws IOExc
     }
 
     protected Translog createTranslog(Path translogPath, LongSupplier primaryTermSupplier) throws IOException {
-        TranslogConfig translogConfig = new TranslogConfig(shardId, translogPath, INDEX_SETTINGS, BigArrays.NON_RECYCLING_INSTANCE, "");
+        TranslogConfig translogConfig = new TranslogConfig(
+            shardId,
+            translogPath,
+            INDEX_SETTINGS,
+            BigArrays.NON_RECYCLING_INSTANCE,
+            "",
+            false
+        );
         String translogUUID = Translog.createEmptyTranslog(
             translogPath,
             SequenceNumbers.NO_OPS_PERFORMED,
diff --git a/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java b/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java
index e34bc078896f9..8b3fc6651a505 100644
--- a/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java
+++ b/server/src/test/java/org/opensearch/index/translog/transfer/TranslogTransferManagerTests.java
@@ -18,6 +18,7 @@
 import org.opensearch.common.blobstore.stream.write.WritePriority;
 import org.opensearch.common.blobstore.support.PlainBlobMetadata;
 import org.opensearch.common.collect.Tuple;
+import org.opensearch.common.unit.TimeValue;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.index.Index;
 import org.opensearch.core.index.shard.ShardId;
@@ -27,6 +28,8 @@
 import org.opensearch.index.translog.transfer.FileSnapshot.TransferFileSnapshot;
 import org.opensearch.index.translog.transfer.FileSnapshot.TranslogFileSnapshot;
 import org.opensearch.index.translog.transfer.listener.TranslogTransferListener;
+import org.opensearch.indices.DefaultRemoteStoreSettings;
+import org.opensearch.indices.RemoteStoreSettings;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.threadpool.TestThreadPool;
 import org.opensearch.threadpool.ThreadPool;
@@ -48,6 +51,8 @@
 
 import org.mockito.Mockito;
 
+import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.TRANSLOG;
+import static org.opensearch.index.remote.RemoteStoreEnums.DataType.METADATA;
 import static org.opensearch.index.translog.transfer.TranslogTransferMetadata.METADATA_SEPARATOR;
 import static org.mockito.ArgumentMatchers.anyInt;
 import static org.mockito.ArgumentMatchers.anyMap;
@@ -95,9 +100,11 @@ public void setUp() throws Exception {
         translogTransferManager = new TranslogTransferManager(
             shardId,
             transferService,
-            remoteBaseTransferPath,
+            remoteBaseTransferPath.add(TRANSLOG.getName()),
+            remoteBaseTransferPath.add(METADATA.getName()),
             tracker,
-            remoteTranslogTransferTracker
+            remoteTranslogTransferTracker,
+            DefaultRemoteStoreSettings.INSTANCE
         );
 
         delayForBlobDownload = 1;
@@ -159,9 +166,11 @@ public void onFailure(TransferFileSnapshot fileSnapshot, Exception e) {
         TranslogTransferManager translogTransferManager = new TranslogTransferManager(
             shardId,
             transferService,
-            remoteBaseTransferPath,
+            remoteBaseTransferPath.add(TRANSLOG.getName()),
+            remoteBaseTransferPath.add(METADATA.getName()),
             fileTransferTracker,
-            remoteTranslogTransferTracker
+            remoteTranslogTransferTracker,
+            DefaultRemoteStoreSettings.INSTANCE
         );
 
         assertTrue(translogTransferManager.transferSnapshot(createTransferSnapshot(), new TranslogTransferListener() {
@@ -184,19 +193,36 @@ public void onUploadFailed(TransferSnapshot transferSnapshot, Exception ex) {
 
     public void testTransferSnapshotOnUploadTimeout() throws Exception {
         doAnswer(invocationOnMock -> {
-            Thread.sleep(31 * 1000);
+            Set<TransferFileSnapshot> transferFileSnapshots = invocationOnMock.getArgument(0);
+            ActionListener<TransferFileSnapshot> listener = invocationOnMock.getArgument(2);
+            Runnable runnable = () -> {
+                try {
+                    Thread.sleep(1000);
+                } catch (InterruptedException e) {
+                    throw new RuntimeException(e);
+                }
+                for (TransferFileSnapshot transferFileSnapshot : transferFileSnapshots) {
+                    listener.onResponse(transferFileSnapshot);
+                }
+            };
+            Thread t = new Thread(runnable);
+            t.start();
             return null;
         }).when(transferService).uploadBlobs(anySet(), anyMap(), any(ActionListener.class), any(WritePriority.class));
         FileTransferTracker fileTransferTracker = new FileTransferTracker(
             new ShardId("index", "indexUUid", 0),
             remoteTranslogTransferTracker
         );
+        RemoteStoreSettings remoteStoreSettings = mock(RemoteStoreSettings.class);
+        when(remoteStoreSettings.getClusterRemoteTranslogTransferTimeout()).thenReturn(new TimeValue(1));
         TranslogTransferManager translogTransferManager = new TranslogTransferManager(
             shardId,
             transferService,
-            remoteBaseTransferPath,
+            remoteBaseTransferPath.add(TRANSLOG.getName()),
+            remoteBaseTransferPath.add(METADATA.getName()),
             fileTransferTracker,
-            remoteTranslogTransferTracker
+            remoteTranslogTransferTracker,
+            remoteStoreSettings
         );
         SetOnce<Exception> exception = new SetOnce<>();
         translogTransferManager.transferSnapshot(createTransferSnapshot(), new TranslogTransferListener() {
@@ -235,9 +261,11 @@ public void testTransferSnapshotOnThreadInterrupt() throws Exception {
         TranslogTransferManager translogTransferManager = new TranslogTransferManager(
             shardId,
             transferService,
-            remoteBaseTransferPath,
+            remoteBaseTransferPath.add(TRANSLOG.getName()),
+            remoteBaseTransferPath.add(METADATA.getName()),
             fileTransferTracker,
-            remoteTranslogTransferTracker
+            remoteTranslogTransferTracker,
+            DefaultRemoteStoreSettings.INSTANCE
         );
         SetOnce<Exception> exception = new SetOnce<>();
 
@@ -330,13 +358,6 @@ public String toString() {
     }
 
     public void testReadMetadataNoFile() throws IOException {
-        TranslogTransferManager translogTransferManager = new TranslogTransferManager(
-            shardId,
-            transferService,
-            remoteBaseTransferPath,
-            null,
-            remoteTranslogTransferTracker
-        );
         doAnswer(invocation -> {
             LatchedActionListener<List<BlobMetadata>> latchedActionListener = invocation.getArgument(3);
             List<BlobMetadata> bmList = new LinkedList<>();
@@ -351,13 +372,6 @@ public void testReadMetadataNoFile() throws IOException {
 
     // This should happen most of the time -
     public void testReadMetadataFile() throws IOException {
-        TranslogTransferManager translogTransferManager = new TranslogTransferManager(
-            shardId,
-            transferService,
-            remoteBaseTransferPath,
-            null,
-            remoteTranslogTransferTracker
-        );
         TranslogTransferMetadata metadata1 = new TranslogTransferMetadata(1, 1, 1, 2);
         String mdFilename1 = metadata1.getFileName();
 
@@ -387,13 +401,6 @@ public void testReadMetadataFile() throws IOException {
     }
 
     public void testReadMetadataReadException() throws IOException {
-        TranslogTransferManager translogTransferManager = new TranslogTransferManager(
-            shardId,
-            transferService,
-            remoteBaseTransferPath,
-            null,
-            remoteTranslogTransferTracker
-        );
         TranslogTransferMetadata tm = new TranslogTransferMetadata(1, 1, 1, 2);
         String mdFilename = tm.getFileName();
 
@@ -423,14 +430,6 @@ public void testMetadataFileNameOrder() throws IOException {
     }
 
     public void testReadMetadataListException() throws IOException {
-        TranslogTransferManager translogTransferManager = new TranslogTransferManager(
-            shardId,
-            transferService,
-            remoteBaseTransferPath,
-            null,
-            remoteTranslogTransferTracker
-        );
-
         doAnswer(invocation -> {
             LatchedActionListener<List<BlobMetadata>> latchedActionListener = invocation.getArgument(3);
             latchedActionListener.onFailure(new IOException("Issue while listing"));
@@ -499,9 +498,11 @@ public void testDeleteTranslogSuccess() throws Exception {
         TranslogTransferManager translogTransferManager = new TranslogTransferManager(
             shardId,
             blobStoreTransferService,
-            remoteBaseTransferPath,
+            remoteBaseTransferPath.add(TRANSLOG.getName()),
+            remoteBaseTransferPath.add(METADATA.getName()),
             tracker,
-            remoteTranslogTransferTracker
+            remoteTranslogTransferTracker,
+            DefaultRemoteStoreSettings.INSTANCE
         );
         String translogFile = "translog-19.tlog", checkpointFile = "translog-19.ckp";
         tracker.add(translogFile, true);
@@ -515,13 +516,6 @@ public void testDeleteTranslogSuccess() throws Exception {
     }
 
     public void testDeleteStaleTranslogMetadata() {
-        TranslogTransferManager translogTransferManager = new TranslogTransferManager(
-            shardId,
-            transferService,
-            remoteBaseTransferPath,
-            null,
-            remoteTranslogTransferTracker
-        );
         String tm1 = new TranslogTransferMetadata(1, 1, 1, 2).getFileName();
         String tm2 = new TranslogTransferMetadata(1, 2, 1, 2).getFileName();
         String tm3 = new TranslogTransferMetadata(2, 3, 1, 2).getFileName();
@@ -569,9 +563,11 @@ public void testDeleteTranslogFailure() throws Exception {
         TranslogTransferManager translogTransferManager = new TranslogTransferManager(
             shardId,
             blobStoreTransferService,
-            remoteBaseTransferPath,
+            remoteBaseTransferPath.add(TRANSLOG.getName()),
+            remoteBaseTransferPath.add(METADATA.getName()),
             tracker,
-            remoteTranslogTransferTracker
+            remoteTranslogTransferTracker,
+            DefaultRemoteStoreSettings.INSTANCE
         );
         String translogFile = "translog-19.tlog", checkpointFile = "translog-19.ckp";
         tracker.add(translogFile, true);
@@ -586,9 +582,7 @@ private void assertNoDownloadStats(boolean nonZeroUploadTime) {
         assertEquals(0, remoteTranslogTransferTracker.getDownloadBytesSucceeded());
         assertEquals(0, remoteTranslogTransferTracker.getTotalDownloadsSucceeded());
         assertEquals(0, remoteTranslogTransferTracker.getLastSuccessfulDownloadTimestamp());
-        if (nonZeroUploadTime) {
-            assertNotEquals(0, remoteTranslogTransferTracker.getTotalDownloadTimeInMillis());
-        } else {
+        if (nonZeroUploadTime == false) {
             assertEquals(0, remoteTranslogTransferTracker.getTotalDownloadTimeInMillis());
         }
     }
@@ -609,13 +603,6 @@ public void testGetPrimaryTermAndGeneration() {
     }
 
     public void testMetadataConflict() throws InterruptedException {
-        TranslogTransferManager translogTransferManager = new TranslogTransferManager(
-            shardId,
-            transferService,
-            remoteBaseTransferPath,
-            null,
-            remoteTranslogTransferTracker
-        );
         TranslogTransferMetadata tm = new TranslogTransferMetadata(1, 1, 1, 2, "node--1");
         String mdFilename = tm.getFileName();
         long count = mdFilename.chars().filter(ch -> ch == METADATA_SEPARATOR.charAt(0)).count();
diff --git a/server/src/test/java/org/opensearch/indices/IRCKeyWriteableSerializerTests.java b/server/src/test/java/org/opensearch/indices/IRCKeyWriteableSerializerTests.java
index af657dadd7a1a..fb5c0a3f9c8f7 100644
--- a/server/src/test/java/org/opensearch/indices/IRCKeyWriteableSerializerTests.java
+++ b/server/src/test/java/org/opensearch/indices/IRCKeyWriteableSerializerTests.java
@@ -30,7 +30,7 @@ public void testSerializer() throws Exception {
         Random rand = Randomness.get();
         for (int valueLength : valueLengths) {
             for (int i = 0; i < NUM_KEYS; i++) {
-                IndicesRequestCache.Key key = getRandomIRCKey(valueLength, rand, indexShard.shardId());
+                IndicesRequestCache.Key key = getRandomIRCKey(valueLength, rand, indexShard.shardId(), System.identityHashCode(indexShard));
                 byte[] serialized = ser.serialize(key);
                 assertTrue(ser.equals(key, serialized));
                 IndicesRequestCache.Key deserialized = ser.deserialize(serialized);
@@ -39,12 +39,13 @@ public void testSerializer() throws Exception {
         }
     }
 
-    private IndicesRequestCache.Key getRandomIRCKey(int valueLength, Random random, ShardId shard) {
+    private IndicesRequestCache.Key getRandomIRCKey(int valueLength, Random random, ShardId shard, int indexShardHashCode) {
         byte[] value = new byte[valueLength];
         for (int i = 0; i < valueLength; i++) {
             value[i] = (byte) (random.nextInt(126 - 32) + 32);
         }
         BytesReference keyValue = new BytesArray(value);
-        return new IndicesRequestCache.Key(shard, keyValue, UUID.randomUUID().toString()); // same UUID source as used in real key
+        return new IndicesRequestCache.Key(shard, keyValue, UUID.randomUUID().toString(), indexShardHashCode); // same UUID
+        // source as used in real key
     }
 }
diff --git a/server/src/test/java/org/opensearch/indices/IndicesLifecycleListenerSingleNodeTests.java b/server/src/test/java/org/opensearch/indices/IndicesLifecycleListenerSingleNodeTests.java
index 0e16e81b1bb70..0428bdf0655b0 100644
--- a/server/src/test/java/org/opensearch/indices/IndicesLifecycleListenerSingleNodeTests.java
+++ b/server/src/test/java/org/opensearch/indices/IndicesLifecycleListenerSingleNodeTests.java
@@ -34,6 +34,7 @@
 import org.opensearch.Version;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.cluster.node.DiscoveryNodes;
 import org.opensearch.cluster.routing.RecoverySource;
 import org.opensearch.cluster.routing.ShardRouting;
 import org.opensearch.cluster.routing.ShardRoutingHelper;
@@ -164,7 +165,8 @@ public void afterIndexRemoved(Index index, IndexSettings indexSettings, IndexRem
                 null,
                 null,
                 localNode,
-                null
+                null,
+                DiscoveryNodes.builder().add(localNode).build()
             );
             IndexShardTestCase.updateRoutingEntry(shard, newRouting);
             assertEquals(5, counter.get());
diff --git a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
index 594b9aac971b7..dcddd9f3d1318 100644
--- a/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
+++ b/server/src/test/java/org/opensearch/indices/IndicesRequestCacheTests.java
@@ -38,27 +38,38 @@
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.BytesRef;
+import org.opensearch.Version;
+import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.cluster.node.DiscoveryNodes;
+import org.opensearch.cluster.routing.RecoverySource;
+import org.opensearch.cluster.routing.ShardRouting;
+import org.opensearch.cluster.routing.ShardRoutingHelper;
+import org.opensearch.cluster.routing.UnassignedInfo;
 import org.opensearch.common.CheckedSupplier;
+import org.opensearch.common.cache.ICacheKey;
 import org.opensearch.common.cache.RemovalNotification;
 import org.opensearch.common.cache.RemovalReason;
 import org.opensearch.common.cache.module.CacheModule;
-import org.opensearch.common.cache.service.CacheService;
+import org.opensearch.common.cache.stats.ImmutableCacheStats;
+import org.opensearch.common.cache.stats.ImmutableCacheStatsHolder;
 import org.opensearch.common.io.stream.BytesStreamOutput;
 import org.opensearch.common.lucene.index.OpenSearchDirectoryReader;
 import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.util.FeatureFlags;
 import org.opensearch.common.util.io.IOUtils;
 import org.opensearch.core.common.bytes.AbstractBytesReference;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.core.common.io.stream.StreamInput;
 import org.opensearch.core.common.unit.ByteSizeValue;
+import org.opensearch.core.index.Index;
 import org.opensearch.core.index.shard.ShardId;
 import org.opensearch.core.xcontent.MediaTypeRegistry;
 import org.opensearch.core.xcontent.XContentHelper;
@@ -67,49 +78,79 @@
 import org.opensearch.index.cache.request.RequestCacheStats;
 import org.opensearch.index.cache.request.ShardRequestCache;
 import org.opensearch.index.query.TermQueryBuilder;
+import org.opensearch.index.seqno.RetentionLeaseSyncer;
 import org.opensearch.index.shard.IndexShard;
 import org.opensearch.index.shard.IndexShardState;
+import org.opensearch.index.shard.IndexShardTestCase;
+import org.opensearch.index.shard.ShardNotFoundException;
+import org.opensearch.indices.replication.checkpoint.SegmentReplicationCheckpointPublisher;
 import org.opensearch.node.Node;
+import org.opensearch.test.ClusterServiceUtils;
 import org.opensearch.test.OpenSearchSingleNodeTestCase;
 import org.opensearch.threadpool.ThreadPool;
+import org.junit.After;
+import org.junit.Before;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.emptySet;
+import static org.opensearch.indices.IndicesRequestCache.INDEX_DIMENSION_NAME;
+import static org.opensearch.indices.IndicesRequestCache.INDICES_CACHE_QUERY_SIZE;
 import static org.opensearch.indices.IndicesRequestCache.INDICES_REQUEST_CACHE_STALENESS_THRESHOLD_SETTING;
+import static org.opensearch.indices.IndicesRequestCache.SHARD_ID_DIMENSION_NAME;
+import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
 public class IndicesRequestCacheTests extends OpenSearchSingleNodeTestCase {
+    private ThreadPool threadPool;
+    private IndexWriter writer;
+    private Directory dir;
+    private IndicesRequestCache cache;
+    private IndexShard indexShard;
+
     private ThreadPool getThreadPool() {
         return new ThreadPool(Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), "default tracer tests").build());
     }
 
-    public void testBasicOperationsCache() throws Exception {
-        IndexShard indexShard = createIndex("test").getShard(0);
-        ThreadPool threadPool = getThreadPool();
-        IndicesRequestCache cache = new IndicesRequestCache(
-            Settings.EMPTY,
-            (shardId -> Optional.of(new IndicesService.IndexShardCacheEntity(indexShard))),
-            new CacheModule(new ArrayList<>(), Settings.EMPTY).getCacheService(),
-            threadPool
-        );
-        Directory dir = newDirectory();
-        IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
+    @Before
+    public void setup() throws IOException {
+        dir = newDirectory();
+        writer = new IndexWriter(dir, newIndexWriterConfig());
+        indexShard = createIndex("test").getShard(0);
+    }
+
+    @After
+    public void cleanup() throws IOException {
+        IOUtils.close(writer, dir, cache);
+        terminate(threadPool);
+    }
 
+    public void testBasicOperationsCache() throws Exception {
+        threadPool = getThreadPool();
+        cache = getIndicesRequestCache(Settings.EMPTY);
         writer.addDocument(newDoc(0, "foo"));
-        DirectoryReader reader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
-        TermQueryBuilder termQuery = new TermQueryBuilder("id", "0");
-        BytesReference termBytes = XContentHelper.toXContent(termQuery, MediaTypeRegistry.JSON, false);
+        DirectoryReader reader = getReader(writer, indexShard.shardId());
 
         // initial cache
         IndicesService.IndexShardCacheEntity entity = new IndicesService.IndexShardCacheEntity(indexShard);
         Loader loader = new Loader(reader, 0);
-        BytesReference value = cache.getOrCompute(entity, loader, reader, termBytes);
+        BytesReference value = cache.getOrCompute(entity, loader, reader, getTermBytes());
         assertEquals("foo", value.streamInput().readString());
         ShardRequestCache requestCacheStats = indexShard.requestCache();
         assertEquals(0, requestCacheStats.stats().getHitCount());
@@ -121,7 +162,7 @@ public void testBasicOperationsCache() throws Exception {
         // cache hit
         entity = new IndicesService.IndexShardCacheEntity(indexShard);
         loader = new Loader(reader, 0);
-        value = cache.getOrCompute(entity, loader, reader, termBytes);
+        value = cache.getOrCompute(entity, loader, reader, getTermBytes());
         assertEquals("foo", value.streamInput().readString());
         requestCacheStats = indexShard.requestCache();
         assertEquals(1, requestCacheStats.stats().getHitCount());
@@ -147,33 +188,21 @@ public void testBasicOperationsCache() throws Exception {
         assertEquals(0, cache.count());
         assertEquals(0, requestCacheStats.stats().getMemorySize().bytesAsInt());
 
-        IOUtils.close(reader, writer, dir, cache);
-        terminate(threadPool);
+        IOUtils.close(reader);
         assertEquals(0, cache.numRegisteredCloseListeners());
     }
 
     public void testBasicOperationsCacheWithFeatureFlag() throws Exception {
-        IndexShard indexShard = createIndex("test").getShard(0);
-        CacheService cacheService = new CacheModule(new ArrayList<>(), Settings.EMPTY).getCacheService();
-        ThreadPool threadPool = getThreadPool();
-        IndicesRequestCache cache = new IndicesRequestCache(
-            Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.PLUGGABLE_CACHE, "true").build(),
-            (shardId -> Optional.of(new IndicesService.IndexShardCacheEntity(indexShard))),
-            cacheService,
-            threadPool
-        );
-        Directory dir = newDirectory();
-        IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
-
+        threadPool = getThreadPool();
+        Settings settings = Settings.builder().put(super.featureFlagSettings()).put(FeatureFlags.PLUGGABLE_CACHE, "true").build();
+        cache = getIndicesRequestCache(settings);
         writer.addDocument(newDoc(0, "foo"));
-        DirectoryReader reader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
-        TermQueryBuilder termQuery = new TermQueryBuilder("id", "0");
-        BytesReference termBytes = XContentHelper.toXContent(termQuery, MediaTypeRegistry.JSON, false);
+        DirectoryReader reader = getReader(writer, indexShard.shardId());
 
         // initial cache
         IndicesService.IndexShardCacheEntity entity = new IndicesService.IndexShardCacheEntity(indexShard);
         Loader loader = new Loader(reader, 0);
-        BytesReference value = cache.getOrCompute(entity, loader, reader, termBytes);
+        BytesReference value = cache.getOrCompute(entity, loader, reader, getTermBytes());
         assertEquals("foo", value.streamInput().readString());
         ShardRequestCache requestCacheStats = indexShard.requestCache();
         assertEquals(0, requestCacheStats.stats().getHitCount());
@@ -185,7 +214,7 @@ public void testBasicOperationsCacheWithFeatureFlag() throws Exception {
         // cache hit
         entity = new IndicesService.IndexShardCacheEntity(indexShard);
         loader = new Loader(reader, 0);
-        value = cache.getOrCompute(entity, loader, reader, termBytes);
+        value = cache.getOrCompute(entity, loader, reader, getTermBytes());
         assertEquals("foo", value.streamInput().readString());
         requestCacheStats = indexShard.requestCache();
         assertEquals(1, requestCacheStats.stats().getHitCount());
@@ -211,43 +240,28 @@ public void testBasicOperationsCacheWithFeatureFlag() throws Exception {
         assertEquals(0, cache.count());
         assertEquals(0, requestCacheStats.stats().getMemorySize().bytesAsInt());
 
-        IOUtils.close(reader, writer, dir, cache);
-        terminate(threadPool);
+        IOUtils.close(reader);
         assertEquals(0, cache.numRegisteredCloseListeners());
     }
 
     public void testCacheDifferentReaders() throws Exception {
-        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexShard indexShard = createIndex("test").getShard(0);
-        ThreadPool threadPool = getThreadPool();
-        IndicesRequestCache cache = new IndicesRequestCache(Settings.EMPTY, (shardId -> {
-            IndexService indexService = null;
-            try {
-                indexService = indicesService.indexServiceSafe(shardId.getIndex());
-            } catch (IndexNotFoundException ex) {
-                return Optional.empty();
-            }
-            return Optional.of(new IndicesService.IndexShardCacheEntity(indexService.getShard(shardId.id())));
-        }), new CacheModule(new ArrayList<>(), Settings.EMPTY).getCacheService(), threadPool);
-        Directory dir = newDirectory();
-        IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
-
+        threadPool = getThreadPool();
+        cache = getIndicesRequestCache(Settings.EMPTY);
         writer.addDocument(newDoc(0, "foo"));
-        DirectoryReader reader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
-        TermQueryBuilder termQuery = new TermQueryBuilder("id", "0");
-        BytesReference termBytes = XContentHelper.toXContent(termQuery, MediaTypeRegistry.JSON, false);
+        DirectoryReader reader = getReader(writer, indexShard.shardId());
+
         if (randomBoolean()) {
             writer.flush();
             IOUtils.close(writer);
             writer = new IndexWriter(dir, newIndexWriterConfig());
         }
         writer.updateDocument(new Term("id", "0"), newDoc(0, "bar"));
-        DirectoryReader secondReader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
+        DirectoryReader secondReader = getReader(writer, indexShard.shardId());
 
         // initial cache
         IndicesService.IndexShardCacheEntity entity = new IndicesService.IndexShardCacheEntity(indexShard);
         Loader loader = new Loader(reader, 0);
-        BytesReference value = cache.getOrCompute(entity, loader, reader, termBytes);
+        BytesReference value = cache.getOrCompute(entity, loader, reader, getTermBytes());
         ShardRequestCache requestCacheStats = entity.stats();
         assertEquals("foo", value.streamInput().readString());
         assertEquals(0, requestCacheStats.stats().getHitCount());
@@ -262,7 +276,7 @@ public void testCacheDifferentReaders() throws Exception {
         // cache the second
         IndicesService.IndexShardCacheEntity secondEntity = new IndicesService.IndexShardCacheEntity(indexShard);
         loader = new Loader(secondReader, 0);
-        value = cache.getOrCompute(entity, loader, secondReader, termBytes);
+        value = cache.getOrCompute(entity, loader, secondReader, getTermBytes());
         requestCacheStats = entity.stats();
         assertEquals("bar", value.streamInput().readString());
         assertEquals(0, requestCacheStats.stats().getHitCount());
@@ -275,7 +289,7 @@ public void testCacheDifferentReaders() throws Exception {
 
         secondEntity = new IndicesService.IndexShardCacheEntity(indexShard);
         loader = new Loader(secondReader, 0);
-        value = cache.getOrCompute(secondEntity, loader, secondReader, termBytes);
+        value = cache.getOrCompute(secondEntity, loader, secondReader, getTermBytes());
         requestCacheStats = entity.stats();
         assertEquals("bar", value.streamInput().readString());
         assertEquals(1, requestCacheStats.stats().getHitCount());
@@ -286,7 +300,7 @@ public void testCacheDifferentReaders() throws Exception {
 
         entity = new IndicesService.IndexShardCacheEntity(indexShard);
         loader = new Loader(reader, 0);
-        value = cache.getOrCompute(entity, loader, reader, termBytes);
+        value = cache.getOrCompute(entity, loader, reader, getTermBytes());
         assertEquals("foo", value.streamInput().readString());
         requestCacheStats = entity.stats();
         assertEquals(2, requestCacheStats.stats().getHitCount());
@@ -319,8 +333,7 @@ public void testCacheDifferentReaders() throws Exception {
         assertEquals(0, cache.count());
         assertEquals(0, requestCacheStats.stats().getMemorySize().bytesAsInt());
 
-        IOUtils.close(secondReader, writer, dir, cache);
-        terminate(threadPool);
+        IOUtils.close(secondReader);
         assertEquals(0, cache.numRegisteredCloseListeners());
     }
 
@@ -347,51 +360,20 @@ public void testCacheCleanupThresholdSettingValidator_Invalid_Percentage() {
         assertThrows(IllegalArgumentException.class, () -> { IndicesRequestCache.validateStalenessSetting("500%"); });
     }
 
+    // when staleness threshold is zero, stale keys should be cleaned up every time cache cleaner is invoked.
     public void testCacheCleanupBasedOnZeroThreshold() throws Exception {
-        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexShard indexShard = createIndex("test").getShard(0);
-        ThreadPool threadPool = getThreadPool();
+        threadPool = getThreadPool();
         Settings settings = Settings.builder().put(INDICES_REQUEST_CACHE_STALENESS_THRESHOLD_SETTING.getKey(), "0%").build();
-        IndicesRequestCache cache = new IndicesRequestCache(settings, (shardId -> {
-            IndexService indexService = null;
-            try {
-                indexService = indicesService.indexServiceSafe(shardId.getIndex());
-            } catch (IndexNotFoundException ex) {
-                return Optional.empty();
-            }
-            return Optional.of(new IndicesService.IndexShardCacheEntity(indexService.getShard(shardId.id())));
-        }), new CacheModule(new ArrayList<>(), settings).getCacheService(), threadPool);
-        Directory dir = newDirectory();
-        IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
-
+        cache = getIndicesRequestCache(settings);
         writer.addDocument(newDoc(0, "foo"));
-        DirectoryReader reader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
-        TermQueryBuilder termQuery = new TermQueryBuilder("id", "0");
-        BytesReference termBytes = XContentHelper.toXContent(termQuery, MediaTypeRegistry.JSON, false);
-        if (randomBoolean()) {
-            writer.flush();
-            IOUtils.close(writer);
-            writer = new IndexWriter(dir, newIndexWriterConfig());
-        }
-        writer.updateDocument(new Term("id", "0"), newDoc(0, "bar"));
-        DirectoryReader secondReader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
+        DirectoryReader reader = getReader(writer, indexShard.shardId());
+        DirectoryReader secondReader = getReader(writer, indexShard.shardId());
 
         // Get 2 entries into the cache
-        IndicesService.IndexShardCacheEntity entity = new IndicesService.IndexShardCacheEntity(indexShard);
-        Loader loader = new Loader(reader, 0);
-        cache.getOrCompute(entity, loader, reader, termBytes);
-
-        entity = new IndicesService.IndexShardCacheEntity(indexShard);
-        loader = new Loader(reader, 0);
-        cache.getOrCompute(entity, loader, reader, termBytes);
-
-        IndicesService.IndexShardCacheEntity secondEntity = new IndicesService.IndexShardCacheEntity(indexShard);
-        loader = new Loader(secondReader, 0);
-        cache.getOrCompute(entity, loader, secondReader, termBytes);
+        cache.getOrCompute(getEntity(indexShard), getLoader(reader), reader, getTermBytes());
+        assertEquals(1, cache.count());
 
-        secondEntity = new IndicesService.IndexShardCacheEntity(indexShard);
-        loader = new Loader(secondReader, 0);
-        cache.getOrCompute(secondEntity, loader, secondReader, termBytes);
+        cache.getOrCompute(getEntity(indexShard), getLoader(secondReader), secondReader, getTermBytes());
         assertEquals(2, cache.count());
 
         // Close the reader, to be enqueued for cleanup
@@ -403,191 +385,153 @@ public void testCacheCleanupBasedOnZeroThreshold() throws Exception {
         cache.cacheCleanupManager.cleanCache();
         // cleanup should remove the stale-key
         assertEquals(1, cache.count());
-
-        IOUtils.close(secondReader, writer, dir, cache);
-        terminate(threadPool);
+        IOUtils.close(secondReader);
     }
 
-    public void testCacheCleanupBasedOnStaleThreshold_StalenessEqualToThreshold() throws Exception {
-        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexShard indexShard = createIndex("test").getShard(0);
-        ThreadPool threadPool = getThreadPool();
-        Settings settings = Settings.builder().put(INDICES_REQUEST_CACHE_STALENESS_THRESHOLD_SETTING.getKey(), "0.5").build();
-        IndicesRequestCache cache = new IndicesRequestCache(settings, (shardId -> {
-            IndexService indexService = null;
-            try {
-                indexService = indicesService.indexServiceSafe(shardId.getIndex());
-            } catch (IndexNotFoundException ex) {
-                return Optional.empty();
-            }
-            return Optional.of(new IndicesService.IndexShardCacheEntity(indexService.getShard(shardId.id())));
-        }), new CacheModule(new ArrayList<>(), settings).getCacheService(), threadPool);
-        Directory dir = newDirectory();
-        IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
+    // when staleness count is higher than stale threshold, stale keys should be cleaned up.
+    public void testCacheCleanupBasedOnStaleThreshold_StalenessHigherThanThreshold() throws Exception {
+        threadPool = getThreadPool();
+        Settings settings = Settings.builder().put(INDICES_REQUEST_CACHE_STALENESS_THRESHOLD_SETTING.getKey(), "0.49").build();
+        cache = getIndicesRequestCache(settings);
 
         writer.addDocument(newDoc(0, "foo"));
-        DirectoryReader reader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
-        TermQueryBuilder termQuery = new TermQueryBuilder("id", "0");
-        BytesReference termBytes = XContentHelper.toXContent(termQuery, MediaTypeRegistry.JSON, false);
-        if (randomBoolean()) {
-            writer.flush();
-            IOUtils.close(writer);
-            writer = new IndexWriter(dir, newIndexWriterConfig());
-        }
-        writer.updateDocument(new Term("id", "0"), newDoc(0, "bar"));
-        DirectoryReader secondReader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
+        DirectoryReader reader = getReader(writer, indexShard.shardId());
+        DirectoryReader secondReader = getReader(writer, indexShard.shardId());
 
         // Get 2 entries into the cache
-        IndicesService.IndexShardCacheEntity entity = new IndicesService.IndexShardCacheEntity(indexShard);
-        Loader loader = new Loader(reader, 0);
-        cache.getOrCompute(entity, loader, reader, termBytes);
+        cache.getOrCompute(getEntity(indexShard), getLoader(reader), reader, getTermBytes());
+        assertEquals(1, cache.count());
 
-        entity = new IndicesService.IndexShardCacheEntity(indexShard);
-        loader = new Loader(reader, 0);
-        cache.getOrCompute(entity, loader, reader, termBytes);
+        cache.getOrCompute(getEntity(indexShard), getLoader(secondReader), secondReader, getTermBytes());
+        assertEquals(2, cache.count());
 
-        IndicesService.IndexShardCacheEntity secondEntity = new IndicesService.IndexShardCacheEntity(indexShard);
-        loader = new Loader(secondReader, 0);
-        cache.getOrCompute(entity, loader, secondReader, termBytes);
+        // no stale keys so far
+        assertEquals(0, cache.cacheCleanupManager.getStaleKeysCount().get());
+        // Close the reader, to be enqueued for cleanup
+        reader.close();
+        // 1 out of 2 keys ie 50% are now stale.
+        assertEquals(1, cache.cacheCleanupManager.getStaleKeysCount().get());
+        // cache count should not be affected
+        assertEquals(2, cache.count());
 
-        secondEntity = new IndicesService.IndexShardCacheEntity(indexShard);
-        loader = new Loader(secondReader, 0);
-        cache.getOrCompute(secondEntity, loader, secondReader, termBytes);
+        // clean cache with 49% staleness threshold
+        cache.cacheCleanupManager.cleanCache();
+        // cleanup should have taken effect with 49% threshold
+        assertEquals(1, cache.count());
+        assertEquals(0, cache.cacheCleanupManager.getStaleKeysCount().get());
+
+        IOUtils.close(secondReader);
+    }
+
+    // when staleness count equal to stale threshold, stale keys should be cleaned up.
+    public void testCacheCleanupBasedOnStaleThreshold_StalenessEqualToThreshold() throws Exception {
+        threadPool = getThreadPool();
+        Settings settings = Settings.builder().put(INDICES_REQUEST_CACHE_STALENESS_THRESHOLD_SETTING.getKey(), "0.5").build();
+        cache = getIndicesRequestCache(settings);
+        writer.addDocument(newDoc(0, "foo"));
+        DirectoryReader reader = getReader(writer, indexShard.shardId());
+        DirectoryReader secondReader = getReader(writer, indexShard.shardId());
+
+        // Get 2 entries into the cache
+        cache.getOrCompute(getEntity(indexShard), getLoader(reader), reader, getTermBytes());
+        assertEquals(1, cache.count());
+
+        cache.getOrCompute(getEntity(indexShard), getLoader(secondReader), secondReader, getTermBytes());
         assertEquals(2, cache.count());
 
         // Close the reader, to be enqueued for cleanup
-        // 1 out of 2 keys ie 50% are now stale.
         reader.close();
+        // 1 out of 2 keys ie 50% are now stale.
+        assertEquals(1, cache.cacheCleanupManager.getStaleKeysCount().get());
         // cache count should not be affected
         assertEquals(2, cache.count());
 
         // clean cache with 50% staleness threshold
         cache.cacheCleanupManager.cleanCache();
         // cleanup should have taken effect
+        assertEquals(0, cache.cacheCleanupManager.getStaleKeysCount().get());
         assertEquals(1, cache.count());
 
-        IOUtils.close(secondReader, writer, dir, cache);
-        terminate(threadPool);
+        IOUtils.close(secondReader);
     }
 
+    // when a cache entry that is Stale is evicted for any reason, we have to deduct the count from our staleness count
     public void testStaleCount_OnRemovalNotificationOfStaleKey_DecrementsStaleCount() throws Exception {
-        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexShard indexShard = createIndex("test").getShard(0);
-        ThreadPool threadPool = getThreadPool();
+        threadPool = getThreadPool();
         Settings settings = Settings.builder().put(INDICES_REQUEST_CACHE_STALENESS_THRESHOLD_SETTING.getKey(), "0.51").build();
-        IndicesRequestCache cache = new IndicesRequestCache(settings, (shardId -> {
-            IndexService indexService = null;
-            try {
-                indexService = indicesService.indexServiceSafe(shardId.getIndex());
-            } catch (IndexNotFoundException ex) {
-                return Optional.empty();
-            }
-            return Optional.of(new IndicesService.IndexShardCacheEntity(indexService.getShard(shardId.id())));
-        }), new CacheModule(new ArrayList<>(), settings).getCacheService(), threadPool);
-        Directory dir = newDirectory();
-        IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
-
+        cache = getIndicesRequestCache(settings);
         writer.addDocument(newDoc(0, "foo"));
-        DirectoryReader reader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
-        TermQueryBuilder termQuery = new TermQueryBuilder("id", "0");
-        BytesReference termBytes = XContentHelper.toXContent(termQuery, MediaTypeRegistry.JSON, false);
-        if (randomBoolean()) {
-            writer.flush();
-            IOUtils.close(writer);
-            writer = new IndexWriter(dir, newIndexWriterConfig());
-        }
-        writer.updateDocument(new Term("id", "0"), newDoc(0, "bar"));
-        DirectoryReader secondReader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
+        ShardId shardId = indexShard.shardId();
+        DirectoryReader reader = getReader(writer, indexShard.shardId());
+        DirectoryReader secondReader = getReader(writer, indexShard.shardId());
 
-        // Get 2 entries into the cache
-        IndicesService.IndexShardCacheEntity entity = new IndicesService.IndexShardCacheEntity(indexShard);
-        Loader loader = new Loader(reader, 0);
-        cache.getOrCompute(entity, loader, reader, termBytes);
-
-        entity = new IndicesService.IndexShardCacheEntity(indexShard);
-        loader = new Loader(reader, 0);
-        cache.getOrCompute(entity, loader, reader, termBytes);
-
-        IndicesService.IndexShardCacheEntity secondEntity = new IndicesService.IndexShardCacheEntity(indexShard);
-        loader = new Loader(secondReader, 0);
-        cache.getOrCompute(entity, loader, secondReader, termBytes);
+        // Get 2 entries into the cache from 2 different readers
+        cache.getOrCompute(getEntity(indexShard), getLoader(reader), reader, getTermBytes());
+        assertEquals(1, cache.count());
 
-        secondEntity = new IndicesService.IndexShardCacheEntity(indexShard);
-        loader = new Loader(secondReader, 0);
-        cache.getOrCompute(secondEntity, loader, secondReader, termBytes);
+        cache.getOrCompute(getEntity(indexShard), getLoader(secondReader), secondReader, getTermBytes());
         assertEquals(2, cache.count());
 
-        // Close the reader, to be enqueued for cleanup
+        // assert no stale keys are accounted so far
+        assertEquals(0, cache.cacheCleanupManager.getStaleKeysCount().get());
+        // Close the reader, this should create a stale key
         reader.close();
-        AtomicInteger staleKeysCount = cache.cacheCleanupManager.getStaleKeysCount();
         // 1 out of 2 keys ie 50% are now stale.
-        assertEquals(1, staleKeysCount.get());
+        assertEquals(1, cache.cacheCleanupManager.getStaleKeysCount().get());
         // cache count should not be affected
         assertEquals(2, cache.count());
 
-        OpenSearchDirectoryReader.DelegatingCacheHelper delegatingCacheHelper =
-            (OpenSearchDirectoryReader.DelegatingCacheHelper) secondReader.getReaderCacheHelper();
-        String readerCacheKeyId = delegatingCacheHelper.getDelegatingCacheKey().getId();
         IndicesRequestCache.Key key = new IndicesRequestCache.Key(
-            ((IndexShard) secondEntity.getCacheIdentity()).shardId(),
-            termBytes,
-            readerCacheKeyId
+            indexShard.shardId(),
+            getTermBytes(),
+            getReaderCacheKeyId(reader),
+            indexShard.hashCode()
+        );
+        // test the mapping
+        ConcurrentMap<ShardId, HashMap<String, Integer>> cleanupKeyToCountMap = cache.cacheCleanupManager.getCleanupKeyToCountMap();
+        // shard id should exist
+        assertTrue(cleanupKeyToCountMap.containsKey(shardId));
+        // reader CacheKeyId should NOT exist
+        assertFalse(cleanupKeyToCountMap.get(shardId).containsKey(getReaderCacheKeyId(reader)));
+        // secondReader CacheKeyId should exist
+        assertTrue(cleanupKeyToCountMap.get(shardId).containsKey(getReaderCacheKeyId(secondReader)));
+
+        cache.onRemoval(
+            new RemovalNotification<ICacheKey<IndicesRequestCache.Key>, BytesReference>(
+                new ICacheKey<>(key),
+                getTermBytes(),
+                RemovalReason.EVICTED
+            )
         );
 
-        cache.onRemoval(new RemovalNotification<IndicesRequestCache.Key, BytesReference>(key, termBytes, RemovalReason.EVICTED));
-        staleKeysCount = cache.cacheCleanupManager.getStaleKeysCount();
+        // test the mapping, it should stay the same
+        // shard id should exist
+        assertTrue(cleanupKeyToCountMap.containsKey(shardId));
+        // reader CacheKeyId should NOT exist
+        assertFalse(cleanupKeyToCountMap.get(shardId).containsKey(getReaderCacheKeyId(reader)));
+        // secondReader CacheKeyId should exist
+        assertTrue(cleanupKeyToCountMap.get(shardId).containsKey(getReaderCacheKeyId(secondReader)));
         // eviction of previous stale key from the cache should decrement staleKeysCount in iRC
-        assertEquals(0, staleKeysCount.get());
+        assertEquals(0, cache.cacheCleanupManager.getStaleKeysCount().get());
 
-        IOUtils.close(secondReader, writer, dir, cache);
-        terminate(threadPool);
+        IOUtils.close(secondReader);
     }
 
-    public void testStaleCount_OnRemovalNotificationOfStaleKey_DoesNotDecrementsStaleCount() throws Exception {
-        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexShard indexShard = createIndex("test").getShard(0);
-        ThreadPool threadPool = getThreadPool();
+    // when a cache entry that is NOT Stale is evicted for any reason, staleness count should NOT be deducted
+    public void testStaleCount_OnRemovalNotificationOfNonStaleKey_DoesNotDecrementsStaleCount() throws Exception {
+        threadPool = getThreadPool();
         Settings settings = Settings.builder().put(INDICES_REQUEST_CACHE_STALENESS_THRESHOLD_SETTING.getKey(), "0.51").build();
-        IndicesRequestCache cache = new IndicesRequestCache(settings, (shardId -> {
-            IndexService indexService = null;
-            try {
-                indexService = indicesService.indexServiceSafe(shardId.getIndex());
-            } catch (IndexNotFoundException ex) {
-                return Optional.empty();
-            }
-            return Optional.of(new IndicesService.IndexShardCacheEntity(indexService.getShard(shardId.id())));
-        }), new CacheModule(new ArrayList<>(), settings).getCacheService(), threadPool);
-        Directory dir = newDirectory();
-        IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
-
+        cache = getIndicesRequestCache(settings);
         writer.addDocument(newDoc(0, "foo"));
-        DirectoryReader reader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
-        TermQueryBuilder termQuery = new TermQueryBuilder("id", "0");
-        BytesReference termBytes = XContentHelper.toXContent(termQuery, MediaTypeRegistry.JSON, false);
-        if (randomBoolean()) {
-            writer.flush();
-            IOUtils.close(writer);
-            writer = new IndexWriter(dir, newIndexWriterConfig());
-        }
-        writer.updateDocument(new Term("id", "0"), newDoc(0, "bar"));
-        DirectoryReader secondReader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
+        ShardId shardId = indexShard.shardId();
+        DirectoryReader reader = getReader(writer, indexShard.shardId());
+        DirectoryReader secondReader = getReader(writer, indexShard.shardId());
 
         // Get 2 entries into the cache
-        IndicesService.IndexShardCacheEntity entity = new IndicesService.IndexShardCacheEntity(indexShard);
-        Loader loader = new Loader(reader, 0);
-        cache.getOrCompute(entity, loader, reader, termBytes);
-
-        entity = new IndicesService.IndexShardCacheEntity(indexShard);
-        loader = new Loader(reader, 0);
-        cache.getOrCompute(entity, loader, reader, termBytes);
-
-        IndicesService.IndexShardCacheEntity secondEntity = new IndicesService.IndexShardCacheEntity(indexShard);
-        loader = new Loader(secondReader, 0);
-        cache.getOrCompute(entity, loader, secondReader, termBytes);
+        cache.getOrCompute(getEntity(indexShard), getLoader(reader), reader, getTermBytes());
+        assertEquals(1, cache.count());
 
-        secondEntity = new IndicesService.IndexShardCacheEntity(indexShard);
-        loader = new Loader(secondReader, 0);
-        cache.getOrCompute(secondEntity, loader, secondReader, termBytes);
+        cache.getOrCompute(getEntity(indexShard), getLoader(secondReader), secondReader, getTermBytes());
         assertEquals(2, cache.count());
 
         // Close the reader, to be enqueued for cleanup
@@ -598,92 +542,263 @@ public void testStaleCount_OnRemovalNotificationOfStaleKey_DoesNotDecrementsStal
         // cache count should not be affected
         assertEquals(2, cache.count());
 
-        OpenSearchDirectoryReader.DelegatingCacheHelper delegatingCacheHelper = (OpenSearchDirectoryReader.DelegatingCacheHelper) reader
-            .getReaderCacheHelper();
-        String readerCacheKeyId = delegatingCacheHelper.getDelegatingCacheKey().getId();
+        // evict entry from second reader (this reader is not closed)
         IndicesRequestCache.Key key = new IndicesRequestCache.Key(
-            ((IndexShard) secondEntity.getCacheIdentity()).shardId(),
-            termBytes,
-            readerCacheKeyId
+            indexShard.shardId(),
+            getTermBytes(),
+            getReaderCacheKeyId(secondReader),
+            indexShard.hashCode()
         );
 
-        cache.onRemoval(new RemovalNotification<IndicesRequestCache.Key, BytesReference>(key, termBytes, RemovalReason.EVICTED));
+        // test the mapping
+        ConcurrentMap<ShardId, HashMap<String, Integer>> cleanupKeyToCountMap = cache.cacheCleanupManager.getCleanupKeyToCountMap();
+        // shard id should exist
+        assertTrue(cleanupKeyToCountMap.containsKey(shardId));
+        // reader CacheKeyId should NOT exist
+        assertFalse(cleanupKeyToCountMap.get(shardId).containsKey(getReaderCacheKeyId(reader)));
+        // secondReader CacheKeyId should exist
+        assertTrue(cleanupKeyToCountMap.get(shardId).containsKey(getReaderCacheKeyId(secondReader)));
+
+        cache.onRemoval(
+            new RemovalNotification<ICacheKey<IndicesRequestCache.Key>, BytesReference>(
+                new ICacheKey<>(key),
+                getTermBytes(),
+                RemovalReason.EVICTED
+            )
+        );
+
+        // test the mapping, shardId entry should be cleaned up
+        // shard id should NOT exist
+        assertFalse(cleanupKeyToCountMap.containsKey(shardId));
+
         staleKeysCount = cache.cacheCleanupManager.getStaleKeysCount();
         // eviction of NON-stale key from the cache should NOT decrement staleKeysCount in iRC
         assertEquals(1, staleKeysCount.get());
 
-        IOUtils.close(secondReader, writer, dir, cache);
-        terminate(threadPool);
+        IOUtils.close(secondReader);
     }
 
-    public void testCacheCleanupBasedOnStaleThreshold_StalenessGreaterThanThreshold() throws Exception {
-        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexShard indexShard = createIndex("test").getShard(0);
-        ThreadPool threadPool = getThreadPool();
-        Settings settings = Settings.builder().put(INDICES_REQUEST_CACHE_STALENESS_THRESHOLD_SETTING.getKey(), "0.49").build();
-        IndicesRequestCache cache = new IndicesRequestCache(settings, (shardId -> {
-            IndexService indexService = null;
-            try {
-                indexService = indicesService.indexServiceSafe(shardId.getIndex());
-            } catch (IndexNotFoundException ex) {
-                return Optional.empty();
-            }
-            return Optional.of(new IndicesService.IndexShardCacheEntity(indexService.getShard(shardId.id())));
-        }), new CacheModule(new ArrayList<>(), settings).getCacheService(), threadPool);
-        Directory dir = newDirectory();
-        IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
+    // when a cache entry that is NOT Stale is evicted WITHOUT its reader closing, we should NOT deduct it from staleness count
+    public void testStaleCount_WithoutReaderClosing_DecrementsStaleCount() throws Exception {
+        threadPool = getThreadPool();
+        Settings settings = Settings.builder().put(INDICES_REQUEST_CACHE_STALENESS_THRESHOLD_SETTING.getKey(), "0.51").build();
+        cache = getIndicesRequestCache(settings);
 
         writer.addDocument(newDoc(0, "foo"));
-        DirectoryReader reader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
-        TermQueryBuilder termQuery = new TermQueryBuilder("id", "0");
-        BytesReference termBytes = XContentHelper.toXContent(termQuery, MediaTypeRegistry.JSON, false);
-        if (randomBoolean()) {
-            writer.flush();
-            IOUtils.close(writer);
-            writer = new IndexWriter(dir, newIndexWriterConfig());
+        DirectoryReader reader = getReader(writer, indexShard.shardId());
+        DirectoryReader secondReader = getReader(writer, indexShard.shardId());
+
+        // Get 2 entries into the cache from 2 different readers
+        cache.getOrCompute(getEntity(indexShard), getLoader(reader), reader, getTermBytes());
+        assertEquals(1, cache.count());
+
+        cache.getOrCompute(getEntity(indexShard), getLoader(secondReader), secondReader, getTermBytes());
+        assertEquals(2, cache.count());
+
+        // no keys are stale
+        assertEquals(0, cache.cacheCleanupManager.getStaleKeysCount().get());
+        // create notification for removal of non-stale entry
+        IndicesRequestCache.Key key = new IndicesRequestCache.Key(
+            indexShard.shardId(),
+            getTermBytes(),
+            getReaderCacheKeyId(reader),
+            indexShard.hashCode()
+        );
+        cache.onRemoval(
+            new RemovalNotification<ICacheKey<IndicesRequestCache.Key>, BytesReference>(
+                new ICacheKey<>(key),
+                getTermBytes(),
+                RemovalReason.EVICTED
+            )
+        );
+        // stale keys count should stay zero
+        assertEquals(0, cache.cacheCleanupManager.getStaleKeysCount().get());
+
+        IOUtils.close(reader, secondReader);
+    }
+
+    // test staleness count based on removal notifications
+    public void testStaleCount_OnRemovalNotifications() throws Exception {
+        threadPool = getThreadPool();
+        Settings settings = Settings.builder().put(INDICES_REQUEST_CACHE_STALENESS_THRESHOLD_SETTING.getKey(), "0.51").build();
+        cache = getIndicesRequestCache(settings);
+
+        writer.addDocument(newDoc(0, "foo"));
+        DirectoryReader reader = getReader(writer, indexShard.shardId());
+
+        // Get 5 entries into the cache
+        int totalKeys = 5;
+        IndicesService.IndexShardCacheEntity entity = null;
+        TermQueryBuilder termQuery = null;
+        BytesReference termBytes = null;
+        for (int i = 1; i <= totalKeys; i++) {
+            termQuery = new TermQueryBuilder("id", "" + i);
+            termBytes = XContentHelper.toXContent(termQuery, MediaTypeRegistry.JSON, false);
+            entity = new IndicesService.IndexShardCacheEntity(indexShard);
+            Loader loader = new Loader(reader, 0);
+            cache.getOrCompute(entity, loader, reader, termBytes);
+            assertEquals(i, cache.count());
         }
-        writer.updateDocument(new Term("id", "0"), newDoc(0, "bar"));
-        DirectoryReader secondReader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
+        // no keys are stale yet
+        assertEquals(0, cache.cacheCleanupManager.getStaleKeysCount().get());
+        // closing the reader should make all keys stale
+        reader.close();
+        assertEquals(totalKeys, cache.cacheCleanupManager.getStaleKeysCount().get());
+
+        String readerCacheKeyId = getReaderCacheKeyId(reader);
+        IndexShard indexShard = (IndexShard) entity.getCacheIdentity();
+        IndicesRequestCache.Key key = new IndicesRequestCache.Key(indexShard.shardId(), termBytes, readerCacheKeyId, indexShard.hashCode());
+
+        int staleCount = cache.cacheCleanupManager.getStaleKeysCount().get();
+        // Notification for Replaced should not deduct the staleCount
+        cache.onRemoval(
+            new RemovalNotification<ICacheKey<IndicesRequestCache.Key>, BytesReference>(
+                new ICacheKey<>(key),
+                getTermBytes(),
+                RemovalReason.REPLACED
+            )
+        );
+        // stale keys count should stay the same
+        assertEquals(staleCount, cache.cacheCleanupManager.getStaleKeysCount().get());
+
+        // Notification for all but Replaced should deduct the staleCount
+        RemovalReason[] reasons = { RemovalReason.INVALIDATED, RemovalReason.EVICTED, RemovalReason.EXPLICIT, RemovalReason.CAPACITY };
+        for (RemovalReason reason : reasons) {
+            cache.onRemoval(
+                new RemovalNotification<ICacheKey<IndicesRequestCache.Key>, BytesReference>(new ICacheKey<>(key), getTermBytes(), reason)
+            );
+            assertEquals(--staleCount, cache.cacheCleanupManager.getStaleKeysCount().get());
+        }
+    }
 
-        // Get 2 entries into the cache
-        IndicesService.IndexShardCacheEntity entity = new IndicesService.IndexShardCacheEntity(indexShard);
-        Loader loader = new Loader(reader, 0);
-        cache.getOrCompute(entity, loader, reader, termBytes);
+    // when staleness count less than the stale threshold, stale keys should NOT be cleaned up.
+    public void testCacheCleanupBasedOnStaleThreshold_StalenessLesserThanThreshold() throws Exception {
+        threadPool = getThreadPool();
+        Settings settings = Settings.builder().put(INDICES_REQUEST_CACHE_STALENESS_THRESHOLD_SETTING.getKey(), "51%").build();
+        cache = getIndicesRequestCache(settings);
 
-        entity = new IndicesService.IndexShardCacheEntity(indexShard);
-        loader = new Loader(reader, 0);
-        cache.getOrCompute(entity, loader, reader, termBytes);
+        writer.addDocument(newDoc(0, "foo"));
+        DirectoryReader reader = getReader(writer, indexShard.shardId());
+        DirectoryReader secondReader = getReader(writer, indexShard.shardId());
 
-        IndicesService.IndexShardCacheEntity secondEntity = new IndicesService.IndexShardCacheEntity(indexShard);
-        loader = new Loader(secondReader, 0);
-        cache.getOrCompute(entity, loader, secondReader, termBytes);
+        // Get 2 entries into the cache
+        cache.getOrCompute(getEntity(indexShard), getLoader(reader), reader, getTermBytes());
+        assertEquals(1, cache.count());
 
-        secondEntity = new IndicesService.IndexShardCacheEntity(indexShard);
-        loader = new Loader(secondReader, 0);
-        cache.getOrCompute(secondEntity, loader, secondReader, termBytes);
+        cache.getOrCompute(getEntity(indexShard), getLoader(secondReader), secondReader, getTermBytes());
         assertEquals(2, cache.count());
 
         // Close the reader, to be enqueued for cleanup
-        // 1 out of 2 keys ie 50% are now stale.
         reader.close();
+        // 1 out of 2 keys ie 50% are now stale.
+        assertEquals(1, cache.cacheCleanupManager.getStaleKeysCount().get());
         // cache count should not be affected
         assertEquals(2, cache.count());
 
-        // clean cache with 49% staleness threshold
+        // clean cache with 51% staleness threshold
         cache.cacheCleanupManager.cleanCache();
-        // cleanup should have taken effect with 49% threshold
+        // cleanup should have been ignored
+        assertEquals(1, cache.cacheCleanupManager.getStaleKeysCount().get());
+        assertEquals(2, cache.count());
+
+        IOUtils.close(secondReader);
+    }
+
+    // test the cleanupKeyToCountMap are set appropriately when both readers are closed
+    public void testCleanupKeyToCountMapAreSetAppropriately() throws Exception {
+        threadPool = getThreadPool();
+        Settings settings = Settings.builder().put(INDICES_REQUEST_CACHE_STALENESS_THRESHOLD_SETTING.getKey(), "0.51").build();
+        cache = getIndicesRequestCache(settings);
+
+        writer.addDocument(newDoc(0, "foo"));
+        ShardId shardId = indexShard.shardId();
+        DirectoryReader reader = getReader(writer, shardId);
+        DirectoryReader secondReader = getReader(writer, shardId);
+
+        // Get 2 entries into the cache from 2 different readers
+        cache.getOrCompute(getEntity(indexShard), getLoader(reader), reader, getTermBytes());
         assertEquals(1, cache.count());
+        // test the mappings
+        ConcurrentMap<ShardId, HashMap<String, Integer>> cleanupKeyToCountMap = cache.cacheCleanupManager.getCleanupKeyToCountMap();
+        assertEquals(1, (int) cleanupKeyToCountMap.get(shardId).get(getReaderCacheKeyId(reader)));
 
-        IOUtils.close(secondReader, writer, dir, cache);
-        terminate(threadPool);
+        cache.getOrCompute(getEntity(indexShard), getLoader(secondReader), secondReader, getTermBytes());
+        // test the mapping
+        assertEquals(2, cache.count());
+        assertEquals(1, (int) cleanupKeyToCountMap.get(shardId).get(getReaderCacheKeyId(secondReader)));
+        // create another entry for the second reader
+        cache.getOrCompute(getEntity(indexShard), getLoader(secondReader), secondReader, getTermBytes("id", "1"));
+        // test the mapping
+        assertEquals(3, cache.count());
+        assertEquals(2, (int) cleanupKeyToCountMap.get(shardId).get(getReaderCacheKeyId(secondReader)));
+
+        // Close the reader, to create stale entries
+        reader.close();
+        // cache count should not be affected
+        assertEquals(3, cache.count());
+        // test the mapping, first reader's entry should be removed from the mapping and accounted for in the staleKeysCount
+        assertFalse(cleanupKeyToCountMap.get(shardId).containsKey(getReaderCacheKeyId(reader)));
+        assertEquals(1, cache.cacheCleanupManager.getStaleKeysCount().get());
+        // second reader's mapping should not be affected
+        assertEquals(2, (int) cleanupKeyToCountMap.get(shardId).get(getReaderCacheKeyId(secondReader)));
+        // send removal notification for first reader
+        IndicesRequestCache.Key key = new IndicesRequestCache.Key(
+            indexShard.shardId(),
+            getTermBytes(),
+            getReaderCacheKeyId(reader),
+            indexShard.hashCode()
+        );
+        cache.onRemoval(
+            new RemovalNotification<ICacheKey<IndicesRequestCache.Key>, BytesReference>(
+                new ICacheKey<>(key),
+                getTermBytes(),
+                RemovalReason.EVICTED
+            )
+        );
+        // test the mapping, it should stay the same
+        assertFalse(cleanupKeyToCountMap.get(shardId).containsKey(getReaderCacheKeyId(reader)));
+        // staleKeysCount should be decremented
+        assertEquals(0, cache.cacheCleanupManager.getStaleKeysCount().get());
+        // second reader's mapping should not be affected
+        assertEquals(2, (int) cleanupKeyToCountMap.get(shardId).get(getReaderCacheKeyId(secondReader)));
+
+        // Without closing the secondReader send removal notification of one of its key
+        key = new IndicesRequestCache.Key(indexShard.shardId(), getTermBytes(), getReaderCacheKeyId(secondReader), indexShard.hashCode());
+        cache.onRemoval(
+            new RemovalNotification<ICacheKey<IndicesRequestCache.Key>, BytesReference>(
+                new ICacheKey<>(key),
+                getTermBytes(),
+                RemovalReason.EVICTED
+            )
+        );
+        // staleKeysCount should be the same as before
+        assertEquals(0, cache.cacheCleanupManager.getStaleKeysCount().get());
+        // secondReader's readerCacheKeyId count should be decremented by 1
+        assertEquals(1, (int) cleanupKeyToCountMap.get(shardId).get(getReaderCacheKeyId(secondReader)));
+        // Without closing the secondReader send removal notification of its last key
+        key = new IndicesRequestCache.Key(indexShard.shardId(), getTermBytes(), getReaderCacheKeyId(secondReader), indexShard.hashCode());
+        cache.onRemoval(
+            new RemovalNotification<ICacheKey<IndicesRequestCache.Key>, BytesReference>(
+                new ICacheKey<>(key),
+                getTermBytes(),
+                RemovalReason.EVICTED
+            )
+        );
+        // staleKeysCount should be the same as before
+        assertEquals(0, cache.cacheCleanupManager.getStaleKeysCount().get());
+        // since all the readers of this shard is closed, the cleanupKeyToCountMap should have no entries
+        assertEquals(0, cleanupKeyToCountMap.size());
+
+        IOUtils.close(secondReader);
     }
 
-    public void testCacheCleanupBasedOnStaleThreshold_StalenessLesserThanThreshold() throws Exception {
+    private DirectoryReader getReader(IndexWriter writer, ShardId shardId) throws IOException {
+        return OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), shardId);
+    }
+
+    private IndicesRequestCache getIndicesRequestCache(Settings settings) {
         IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexShard indexShard = createIndex("test").getShard(0);
-        ThreadPool threadPool = getThreadPool();
-        Settings settings = Settings.builder().put(INDICES_REQUEST_CACHE_STALENESS_THRESHOLD_SETTING.getKey(), "51%").build();
-        IndicesRequestCache cache = new IndicesRequestCache(settings, (shardId -> {
+        return new IndicesRequestCache(settings, (shardId -> {
             IndexService indexService = null;
             try {
                 indexService = indicesService.indexServiceSafe(shardId.getIndex());
@@ -691,12 +806,79 @@ public void testCacheCleanupBasedOnStaleThreshold_StalenessLesserThanThreshold()
                 return Optional.empty();
             }
             return Optional.of(new IndicesService.IndexShardCacheEntity(indexService.getShard(shardId.id())));
-        }), new CacheModule(new ArrayList<>(), Settings.EMPTY).getCacheService(), threadPool);
-        Directory dir = newDirectory();
-        IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
+        }),
+            new CacheModule(new ArrayList<>(), Settings.EMPTY).getCacheService(),
+            threadPool,
+            ClusterServiceUtils.createClusterService(threadPool)
+        );
+    }
+
+    private Loader getLoader(DirectoryReader reader) {
+        return new Loader(reader, 0);
+    }
+
+    private IndicesService.IndexShardCacheEntity getEntity(IndexShard indexShard) {
+        return new IndicesService.IndexShardCacheEntity(indexShard);
+    }
+
+    private BytesReference getTermBytes() throws IOException {
+        TermQueryBuilder termQuery = new TermQueryBuilder("id", "0");
+        return XContentHelper.toXContent(termQuery, MediaTypeRegistry.JSON, false);
+    }
+
+    private BytesReference getTermBytes(String fieldName, String value) throws IOException {
+        TermQueryBuilder termQuery = new TermQueryBuilder(fieldName, value);
+        return XContentHelper.toXContent(termQuery, MediaTypeRegistry.JSON, false);
+    }
+
+    private String getReaderCacheKeyId(DirectoryReader reader) {
+        OpenSearchDirectoryReader.DelegatingCacheHelper delegatingCacheHelper = (OpenSearchDirectoryReader.DelegatingCacheHelper) reader
+            .getReaderCacheHelper();
+        return delegatingCacheHelper.getDelegatingCacheKey().getId();
+    }
+
+    public void testClosingIndexWipesStats() throws Exception {
+        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
+        String[] levels = { INDEX_DIMENSION_NAME, SHARD_ID_DIMENSION_NAME };
+        // Create two indices each with multiple shards
+        int numShards = 3;
+        Settings indexSettings = Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, numShards).build();
+        String indexToKeepName = "test";
+        String indexToCloseName = "test2";
+        // delete all indices if already
+        assertAcked(client().admin().indices().prepareDelete("_all").get());
+        IndexService indexToKeep = createIndex(indexToKeepName, indexSettings);
+        IndexService indexToClose = createIndex(indexToCloseName, indexSettings);
+        for (int i = 0; i < numShards; i++) {
+            // Check we can get all the shards we expect
+            assertNotNull(indexToKeep.getShard(i));
+            assertNotNull(indexToClose.getShard(i));
+        }
+
+        threadPool = getThreadPool();
+        Settings settings = Settings.builder()
+            .put(INDICES_REQUEST_CACHE_STALENESS_THRESHOLD_SETTING.getKey(), "0.001%")
+            .put(FeatureFlags.PLUGGABLE_CACHE, true)
+            .build();
+        cache = new IndicesRequestCache(settings, (shardId -> {
+            IndexService indexService = null;
+            try {
+                indexService = indicesService.indexServiceSafe(shardId.getIndex());
+            } catch (IndexNotFoundException ex) {
+                return Optional.empty();
+            }
+            try {
+                return Optional.of(new IndicesService.IndexShardCacheEntity(indexService.getShard(shardId.id())));
+            } catch (ShardNotFoundException ex) {
+                return Optional.empty();
+            }
+        }),
+            new CacheModule(new ArrayList<>(), Settings.EMPTY).getCacheService(),
+            threadPool,
+            ClusterServiceUtils.createClusterService(threadPool)
+        );
 
         writer.addDocument(newDoc(0, "foo"));
-        DirectoryReader reader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
         TermQueryBuilder termQuery = new TermQueryBuilder("id", "0");
         BytesReference termBytes = XContentHelper.toXContent(termQuery, MediaTypeRegistry.JSON, false);
         if (randomBoolean()) {
@@ -707,22 +889,82 @@ public void testCacheCleanupBasedOnStaleThreshold_StalenessLesserThanThreshold()
         writer.updateDocument(new Term("id", "0"), newDoc(0, "bar"));
         DirectoryReader secondReader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
 
-        // Get 2 entries into the cache
-        IndicesService.IndexShardCacheEntity entity = new IndicesService.IndexShardCacheEntity(indexShard);
-        Loader loader = new Loader(reader, 0);
-        cache.getOrCompute(entity, loader, reader, termBytes);
+        List<DirectoryReader> readersToClose = new ArrayList<>();
+        List<DirectoryReader> readersToKeep = new ArrayList<>();
+        // Put entries into the cache for each shard
+        for (IndexService indexService : new IndexService[] { indexToKeep, indexToClose }) {
+            for (int i = 0; i < numShards; i++) {
+                IndexShard indexShard = indexService.getShard(i);
+                IndicesService.IndexShardCacheEntity entity = new IndicesService.IndexShardCacheEntity(indexShard);
+                DirectoryReader reader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), indexShard.shardId());
+                if (indexService == indexToClose) {
+                    readersToClose.add(reader);
+                } else {
+                    readersToKeep.add(reader);
+                }
+                Loader loader = new Loader(reader, 0);
+                cache.getOrCompute(entity, loader, reader, termBytes);
+            }
+        }
 
-        entity = new IndicesService.IndexShardCacheEntity(indexShard);
-        loader = new Loader(reader, 0);
-        cache.getOrCompute(entity, loader, reader, termBytes);
+        // Check resulting stats
+        List<List<String>> initialDimensionValues = new ArrayList<>();
+        for (IndexService indexService : new IndexService[] { indexToKeep, indexToClose }) {
+            for (int i = 0; i < numShards; i++) {
+                ShardId shardId = indexService.getShard(i).shardId();
+                List<String> dimensionValues = List.of(shardId.getIndexName(), shardId.toString());
+                initialDimensionValues.add(dimensionValues);
+                ImmutableCacheStatsHolder holder = cache.stats(levels);
+                ImmutableCacheStats snapshot = cache.stats(levels).getStatsForDimensionValues(dimensionValues);
+                assertNotNull(snapshot);
+                // check the values are not empty by confirming entries != 0, this should always be true since the missed value is loaded
+                // into the cache
+                assertNotEquals(0, snapshot.getItems());
+            }
+        }
 
-        IndicesService.IndexShardCacheEntity secondEntity = new IndicesService.IndexShardCacheEntity(indexShard);
-        loader = new Loader(secondReader, 0);
-        cache.getOrCompute(entity, loader, secondReader, termBytes);
+        // Delete an index
+        indexToClose.close("test_deletion", true);
+        // This actually closes the shards associated with the readers, which is necessary for cache cleanup logic
+        // In this UT, manually close the readers as well; could not figure out how to connect all this up in a UT so that
+        // we could get readers that were properly connected to an index's directory
+        for (DirectoryReader reader : readersToClose) {
+            IOUtils.close(reader);
+        }
+        // Trigger cache cleanup
+        cache.cacheCleanupManager.cleanCache();
 
-        secondEntity = new IndicesService.IndexShardCacheEntity(indexShard);
-        loader = new Loader(secondReader, 0);
-        cache.getOrCompute(secondEntity, loader, secondReader, termBytes);
+        // Now stats for the closed index should be gone
+        for (List<String> dimensionValues : initialDimensionValues) {
+            ImmutableCacheStats snapshot = cache.stats(levels).getStatsForDimensionValues(dimensionValues);
+            if (dimensionValues.get(0).equals(indexToCloseName)) {
+                assertNull(snapshot);
+            } else {
+                assertNotNull(snapshot);
+                // check the values are not empty by confirming entries != 0, this should always be true since the missed value is loaded
+                // into the cache
+                assertNotEquals(0, snapshot.getItems());
+            }
+        }
+
+        for (DirectoryReader reader : readersToKeep) {
+            IOUtils.close(reader);
+        }
+        IOUtils.close(secondReader);
+    }
+
+    public void testCacheCleanupBasedOnStaleThreshold_thresholdUpdate() throws Exception {
+        threadPool = getThreadPool();
+        Settings settings = Settings.builder().put(INDICES_REQUEST_CACHE_STALENESS_THRESHOLD_SETTING.getKey(), "51%").build();
+        cache = getIndicesRequestCache(settings);
+
+        writer.addDocument(newDoc(0, "foo"));
+        DirectoryReader reader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
+        DirectoryReader secondReader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
+
+        // Get 2 entries into the cache
+        cache.getOrCompute(getEntity(indexShard), getLoader(reader), reader, getTermBytes());
+        cache.getOrCompute(getEntity(indexShard), getLoader(secondReader), secondReader, getTermBytes());
         assertEquals(2, cache.count());
 
         // Close the reader, to be enqueued for cleanup
@@ -736,125 +978,88 @@ public void testCacheCleanupBasedOnStaleThreshold_StalenessLesserThanThreshold()
         // cleanup should have been ignored
         assertEquals(2, cache.count());
 
-        IOUtils.close(secondReader, writer, dir, cache);
-        terminate(threadPool);
+        cache.setStalenessThreshold("49%");
+        // clean cache with 49% staleness threshold
+        cache.cacheCleanupManager.cleanCache();
+        // cleanup should NOT have been ignored
+        assertEquals(1, cache.count());
+
+        IOUtils.close(secondReader);
     }
 
     public void testEviction() throws Exception {
         final ByteSizeValue size;
         {
-            IndexShard indexShard = createIndex("test").getShard(0);
-            ThreadPool threadPool = getThreadPool();
-            IndicesRequestCache cache = new IndicesRequestCache(
-                Settings.EMPTY,
-                (shardId -> Optional.of(new IndicesService.IndexShardCacheEntity(indexShard))),
-                new CacheModule(new ArrayList<>(), Settings.EMPTY).getCacheService(),
-                threadPool
-            );
-            Directory dir = newDirectory();
-            IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
-
+            threadPool = getThreadPool();
+            cache = getIndicesRequestCache(Settings.EMPTY);
             writer.addDocument(newDoc(0, "foo"));
-            DirectoryReader reader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
-            TermQueryBuilder termQuery = new TermQueryBuilder("id", "0");
-            BytesReference termBytes = XContentHelper.toXContent(termQuery, MediaTypeRegistry.JSON, false);
-            IndicesService.IndexShardCacheEntity entity = new IndicesService.IndexShardCacheEntity(indexShard);
-            Loader loader = new Loader(reader, 0);
-
+            DirectoryReader reader = getReader(writer, indexShard.shardId());
             writer.updateDocument(new Term("id", "0"), newDoc(0, "bar"));
             DirectoryReader secondReader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
-            IndicesService.IndexShardCacheEntity secondEntity = new IndicesService.IndexShardCacheEntity(indexShard);
-            Loader secondLoader = new Loader(secondReader, 0);
 
-            BytesReference value1 = cache.getOrCompute(entity, loader, reader, termBytes);
+            BytesReference value1 = cache.getOrCompute(getEntity(indexShard), getLoader(reader), reader, getTermBytes());
             assertEquals("foo", value1.streamInput().readString());
-            BytesReference value2 = cache.getOrCompute(secondEntity, secondLoader, secondReader, termBytes);
+            BytesReference value2 = cache.getOrCompute(getEntity(indexShard), getLoader(secondReader), secondReader, getTermBytes());
             assertEquals("bar", value2.streamInput().readString());
-            size = indexShard.requestCache().stats().getMemorySize();
+            size = indexShard.requestCache().stats().getMemorySize(); // Value from old API
             IOUtils.close(reader, secondReader, writer, dir, cache);
-            terminate(threadPool);
         }
-        IndexShard indexShard = createIndex("test1").getShard(0);
-        ThreadPool threadPool = getThreadPool();
+        indexShard = createIndex("test1").getShard(0);
         IndicesRequestCache cache = new IndicesRequestCache(
-            Settings.builder().put(IndicesRequestCache.INDICES_CACHE_QUERY_SIZE.getKey(), size.getBytes() + 1 + "b").build(),
+            // TODO: Add wiggle room to max size to allow for overhead of ICacheKey. This can be removed once API PR goes in, as it updates
+            // the old API to account for the ICacheKey overhead.
+            Settings.builder().put(IndicesRequestCache.INDICES_CACHE_QUERY_SIZE.getKey(), (int) (size.getBytes() * 1.2) + "b").build(),
             (shardId -> Optional.of(new IndicesService.IndexShardCacheEntity(indexShard))),
             new CacheModule(new ArrayList<>(), Settings.EMPTY).getCacheService(),
-            threadPool
+            threadPool,
+            ClusterServiceUtils.createClusterService(threadPool)
         );
-        Directory dir = newDirectory();
-        IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
-
+        dir = newDirectory();
+        writer = new IndexWriter(dir, newIndexWriterConfig());
         writer.addDocument(newDoc(0, "foo"));
-        DirectoryReader reader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
-        TermQueryBuilder termQuery = new TermQueryBuilder("id", "0");
-        BytesReference termBytes = XContentHelper.toXContent(termQuery, MediaTypeRegistry.JSON, false);
-        IndicesService.IndexShardCacheEntity entity = new IndicesService.IndexShardCacheEntity(indexShard);
-        Loader loader = new Loader(reader, 0);
-
+        DirectoryReader reader = getReader(writer, indexShard.shardId());
         writer.updateDocument(new Term("id", "0"), newDoc(0, "bar"));
-        DirectoryReader secondReader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
-        IndicesService.IndexShardCacheEntity secondEntity = new IndicesService.IndexShardCacheEntity(indexShard);
-        Loader secondLoader = new Loader(secondReader, 0);
-
+        DirectoryReader secondReader = getReader(writer, indexShard.shardId());
         writer.updateDocument(new Term("id", "0"), newDoc(0, "baz"));
         DirectoryReader thirdReader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
-        IndicesService.IndexShardCacheEntity thirddEntity = new IndicesService.IndexShardCacheEntity(indexShard);
-        Loader thirdLoader = new Loader(thirdReader, 0);
 
-        BytesReference value1 = cache.getOrCompute(entity, loader, reader, termBytes);
+        BytesReference value1 = cache.getOrCompute(getEntity(indexShard), getLoader(reader), reader, getTermBytes());
         assertEquals("foo", value1.streamInput().readString());
-        BytesReference value2 = cache.getOrCompute(secondEntity, secondLoader, secondReader, termBytes);
+        BytesReference value2 = cache.getOrCompute(getEntity(indexShard), getLoader(secondReader), secondReader, getTermBytes());
         assertEquals("bar", value2.streamInput().readString());
         logger.info("Memory size: {}", indexShard.requestCache().stats().getMemorySize());
-        BytesReference value3 = cache.getOrCompute(thirddEntity, thirdLoader, thirdReader, termBytes);
+        BytesReference value3 = cache.getOrCompute(getEntity(indexShard), getLoader(thirdReader), thirdReader, getTermBytes());
         assertEquals("baz", value3.streamInput().readString());
         assertEquals(2, cache.count());
         assertEquals(1, indexShard.requestCache().stats().getEvictions());
-        IOUtils.close(reader, secondReader, thirdReader, writer, dir, cache);
-        terminate(threadPool);
+        IOUtils.close(reader, secondReader, thirdReader);
     }
 
     public void testClearAllEntityIdentity() throws Exception {
-        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexShard indexShard = createIndex("test").getShard(0);
-        ThreadPool threadPool = getThreadPool();
-        IndicesRequestCache cache = new IndicesRequestCache(Settings.EMPTY, (shardId -> {
-            IndexService indexService = null;
-            try {
-                indexService = indicesService.indexServiceSafe(shardId.getIndex());
-            } catch (IndexNotFoundException ex) {
-                return Optional.empty();
-            }
-            return Optional.of(new IndicesService.IndexShardCacheEntity(indexService.getShard(shardId.id())));
-        }), new CacheModule(new ArrayList<>(), Settings.EMPTY).getCacheService(), threadPool);
-
-        Directory dir = newDirectory();
-        IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
-
+        threadPool = getThreadPool();
+        cache = getIndicesRequestCache(Settings.EMPTY);
         writer.addDocument(newDoc(0, "foo"));
-        DirectoryReader reader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
-        TermQueryBuilder termQuery = new TermQueryBuilder("id", "0");
-        BytesReference termBytes = XContentHelper.toXContent(termQuery, MediaTypeRegistry.JSON, false);
+        DirectoryReader reader = getReader(writer, indexShard.shardId());
         IndicesService.IndexShardCacheEntity entity = new IndicesService.IndexShardCacheEntity(indexShard);
         Loader loader = new Loader(reader, 0);
 
         writer.updateDocument(new Term("id", "0"), newDoc(0, "bar"));
-        DirectoryReader secondReader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
+        DirectoryReader secondReader = getReader(writer, indexShard.shardId());
         IndicesService.IndexShardCacheEntity secondEntity = new IndicesService.IndexShardCacheEntity(indexShard);
         Loader secondLoader = new Loader(secondReader, 0);
 
         writer.updateDocument(new Term("id", "0"), newDoc(0, "baz"));
-        DirectoryReader thirdReader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
+        DirectoryReader thirdReader = getReader(writer, indexShard.shardId());
+        ;
         IndicesService.IndexShardCacheEntity thirddEntity = new IndicesService.IndexShardCacheEntity(createIndex("test1").getShard(0));
         Loader thirdLoader = new Loader(thirdReader, 0);
 
-        BytesReference value1 = cache.getOrCompute(entity, loader, reader, termBytes);
+        BytesReference value1 = cache.getOrCompute(entity, loader, reader, getTermBytes());
         assertEquals("foo", value1.streamInput().readString());
-        BytesReference value2 = cache.getOrCompute(secondEntity, secondLoader, secondReader, termBytes);
+        BytesReference value2 = cache.getOrCompute(secondEntity, secondLoader, secondReader, getTermBytes());
         assertEquals("bar", value2.streamInput().readString());
         logger.info("Memory size: {}", indexShard.requestCache().stats().getMemorySize());
-        BytesReference value3 = cache.getOrCompute(thirddEntity, thirdLoader, thirdReader, termBytes);
+        BytesReference value3 = cache.getOrCompute(thirddEntity, thirdLoader, thirdReader, getTermBytes());
         assertEquals("baz", value3.streamInput().readString());
         assertEquals(3, cache.count());
         RequestCacheStats requestCacheStats = entity.stats().stats();
@@ -865,14 +1070,13 @@ public void testClearAllEntityIdentity() throws Exception {
         cache.cacheCleanupManager.cleanCache();
         assertEquals(1, cache.count());
         // third has not been validated since it's a different identity
-        value3 = cache.getOrCompute(thirddEntity, thirdLoader, thirdReader, termBytes);
+        value3 = cache.getOrCompute(thirddEntity, thirdLoader, thirdReader, getTermBytes());
         requestCacheStats = entity.stats().stats();
         requestCacheStats.add(thirddEntity.stats().stats());
         assertEquals(hitCount + 1, requestCacheStats.getHitCount());
         assertEquals("baz", value3.streamInput().readString());
 
-        IOUtils.close(reader, secondReader, thirdReader, writer, dir, cache);
-        terminate(threadPool);
+        IOUtils.close(reader, secondReader, thirdReader);
     }
 
     public Iterable<Field> newDoc(int id, String value) {
@@ -884,7 +1088,7 @@ public Iterable<Field> newDoc(int id, String value) {
 
     private static class Loader implements CheckedSupplier<BytesReference, IOException> {
 
-        private final DirectoryReader reader;
+        final DirectoryReader reader;
         private final int id;
         public boolean loadedFromCache = true;
 
@@ -908,34 +1112,18 @@ public BytesReference get() {
                 throw new RuntimeException(e);
             }
         }
-
     }
 
     public void testInvalidate() throws Exception {
-        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        IndexShard indexShard = createIndex("test").getShard(0);
-        ThreadPool threadPool = getThreadPool();
-        IndicesRequestCache cache = new IndicesRequestCache(Settings.EMPTY, (shardId -> {
-            IndexService indexService = null;
-            try {
-                indexService = indicesService.indexServiceSafe(shardId.getIndex());
-            } catch (IndexNotFoundException ex) {
-                return Optional.empty();
-            }
-            return Optional.of(new IndicesService.IndexShardCacheEntity(indexService.getShard(shardId.id())));
-        }), new CacheModule(new ArrayList<>(), Settings.EMPTY).getCacheService(), threadPool);
-        Directory dir = newDirectory();
-        IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
-
+        threadPool = getThreadPool();
+        IndicesRequestCache cache = getIndicesRequestCache(Settings.EMPTY);
         writer.addDocument(newDoc(0, "foo"));
-        DirectoryReader reader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
-        TermQueryBuilder termQuery = new TermQueryBuilder("id", "0");
-        BytesReference termBytes = XContentHelper.toXContent(termQuery, MediaTypeRegistry.JSON, false);
+        DirectoryReader reader = getReader(writer, indexShard.shardId());
 
         // initial cache
         IndicesService.IndexShardCacheEntity entity = new IndicesService.IndexShardCacheEntity(indexShard);
         Loader loader = new Loader(reader, 0);
-        BytesReference value = cache.getOrCompute(entity, loader, reader, termBytes);
+        BytesReference value = cache.getOrCompute(entity, loader, reader, getTermBytes());
         assertEquals("foo", value.streamInput().readString());
         ShardRequestCache requestCacheStats = entity.stats();
         assertEquals(0, requestCacheStats.stats().getHitCount());
@@ -947,7 +1135,7 @@ public void testInvalidate() throws Exception {
         // cache hit
         entity = new IndicesService.IndexShardCacheEntity(indexShard);
         loader = new Loader(reader, 0);
-        value = cache.getOrCompute(entity, loader, reader, termBytes);
+        value = cache.getOrCompute(entity, loader, reader, getTermBytes());
         assertEquals("foo", value.streamInput().readString());
         requestCacheStats = entity.stats();
         assertEquals(1, requestCacheStats.stats().getHitCount());
@@ -961,8 +1149,8 @@ public void testInvalidate() throws Exception {
         // load again after invalidate
         entity = new IndicesService.IndexShardCacheEntity(indexShard);
         loader = new Loader(reader, 0);
-        cache.invalidate(entity, reader, termBytes);
-        value = cache.getOrCompute(entity, loader, reader, termBytes);
+        cache.invalidate(entity, reader, getTermBytes());
+        value = cache.getOrCompute(entity, loader, reader, getTermBytes());
         assertEquals("foo", value.streamInput().readString());
         requestCacheStats = entity.stats();
         assertEquals(1, requestCacheStats.stats().getHitCount());
@@ -987,16 +1175,11 @@ public void testInvalidate() throws Exception {
         assertEquals(0, cache.count());
         assertEquals(0, requestCacheStats.stats().getMemorySize().bytesAsInt());
 
-        IOUtils.close(reader, writer, dir, cache);
-        terminate(threadPool);
+        IOUtils.close(reader);
         assertEquals(0, cache.numRegisteredCloseListeners());
     }
 
     public void testEqualsKey() throws IOException {
-        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
-        Directory dir = newDirectory();
-        IndexWriterConfig config = newIndexWriterConfig();
-        IndexWriter writer = new IndexWriter(dir, config);
         ShardId shardId = new ShardId("foo", "bar", 1);
         ShardId shardId1 = new ShardId("foo1", "bar1", 2);
         IndexReader reader1 = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), shardId);
@@ -1007,11 +1190,11 @@ public void testEqualsKey() throws IOException {
         IOUtils.close(reader1, reader2, writer, dir);
         IndexShard indexShard = mock(IndexShard.class);
         when(indexShard.state()).thenReturn(IndexShardState.STARTED);
-        IndicesRequestCache.Key key1 = new IndicesRequestCache.Key(shardId, new TestBytesReference(1), rKey1);
-        IndicesRequestCache.Key key2 = new IndicesRequestCache.Key(shardId, new TestBytesReference(1), rKey1);
-        IndicesRequestCache.Key key3 = new IndicesRequestCache.Key(shardId1, new TestBytesReference(1), rKey1);
-        IndicesRequestCache.Key key4 = new IndicesRequestCache.Key(shardId, new TestBytesReference(1), rKey2);
-        IndicesRequestCache.Key key5 = new IndicesRequestCache.Key(shardId, new TestBytesReference(2), rKey2);
+        IndicesRequestCache.Key key1 = new IndicesRequestCache.Key(shardId, new TestBytesReference(1), rKey1, shardId.hashCode());
+        IndicesRequestCache.Key key2 = new IndicesRequestCache.Key(shardId, new TestBytesReference(1), rKey1, shardId.hashCode());
+        IndicesRequestCache.Key key3 = new IndicesRequestCache.Key(shardId1, new TestBytesReference(1), rKey1, shardId1.hashCode());
+        IndicesRequestCache.Key key4 = new IndicesRequestCache.Key(shardId, new TestBytesReference(1), rKey2, shardId.hashCode());
+        IndicesRequestCache.Key key5 = new IndicesRequestCache.Key(shardId, new TestBytesReference(2), rKey2, shardId.hashCode());
         String s = "Some other random object";
         assertEquals(key1, key1);
         assertEquals(key1, key2);
@@ -1023,13 +1206,14 @@ public void testEqualsKey() throws IOException {
     }
 
     public void testSerializationDeserializationOfCacheKey() throws Exception {
-        TermQueryBuilder termQuery = new TermQueryBuilder("id", "0");
-        BytesReference termBytes = XContentHelper.toXContent(termQuery, MediaTypeRegistry.JSON, false);
-        IndexService indexService = createIndex("test");
-        IndexShard indexShard = indexService.getShard(0);
         IndicesService.IndexShardCacheEntity shardCacheEntity = new IndicesService.IndexShardCacheEntity(indexShard);
         String readerCacheKeyId = UUID.randomUUID().toString();
-        IndicesRequestCache.Key key1 = new IndicesRequestCache.Key(indexShard.shardId(), termBytes, readerCacheKeyId);
+        IndicesRequestCache.Key key1 = new IndicesRequestCache.Key(
+            indexShard.shardId(),
+            getTermBytes(),
+            readerCacheKeyId,
+            indexShard.hashCode()
+        );
         BytesReference bytesReference = null;
         try (BytesStreamOutput out = new BytesStreamOutput()) {
             key1.writeTo(out);
@@ -1041,8 +1225,208 @@ public void testSerializationDeserializationOfCacheKey() throws Exception {
 
         assertEquals(readerCacheKeyId, key2.readerCacheKeyId);
         assertEquals(((IndexShard) shardCacheEntity.getCacheIdentity()).shardId(), key2.shardId);
-        assertEquals(termBytes, key2.value);
+        assertEquals(getTermBytes(), key2.value);
+        assertEquals(indexShard.hashCode(), key2.indexShardHashCode);
+    }
+
+    public void testGetOrComputeConcurrentlyWithMultipleIndices() throws Exception {
+        threadPool = getThreadPool();
+        int numberOfIndices = randomIntBetween(2, 5);
+        List<String> indicesList = new ArrayList<>();
+        List<IndexShard> indexShardList = Collections.synchronizedList(new ArrayList<>());
+        for (int i = 0; i < numberOfIndices; i++) {
+            String indexName = "test" + i;
+            indicesList.add(indexName);
+            IndexShard indexShard = createIndex(
+                indexName,
+                Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build()
+            ).getShard(0);
+            indexShardList.add(indexShard);
+        }
+        // Create a cache with 2kb to cause evictions and test that flow as well.
+        IndicesRequestCache cache = getIndicesRequestCache(Settings.builder().put(INDICES_CACHE_QUERY_SIZE.getKey(), "2kb").build());
+        Map<IndexShard, DirectoryReader> readerMap = new ConcurrentHashMap<>();
+        Map<IndexShard, IndicesService.IndexShardCacheEntity> entityMap = new ConcurrentHashMap<>();
+        Map<IndexShard, IndexWriter> writerMap = new ConcurrentHashMap<>();
+        int numberOfItems = randomIntBetween(200, 400);
+        for (int i = 0; i < numberOfIndices; i++) {
+            IndexShard indexShard = indexShardList.get(i);
+            entityMap.put(indexShard, new IndicesService.IndexShardCacheEntity(indexShard));
+            Directory dir = newDirectory();
+            IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
+            for (int j = 0; j < numberOfItems; j++) {
+                writer.addDocument(newDoc(j, generateString(randomIntBetween(4, 50))));
+            }
+            writerMap.put(indexShard, writer);
+            DirectoryReader reader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), indexShard.shardId());
+            readerMap.put(indexShard, reader);
+        }
+
+        CountDownLatch latch = new CountDownLatch(numberOfItems);
+        ExecutorService executorService = Executors.newFixedThreadPool(5);
+        for (int i = 0; i < numberOfItems; i++) {
+            int finalI = i;
+            executorService.submit(() -> {
+                int randomIndexPosition = randomIntBetween(0, numberOfIndices - 1);
+                IndexShard indexShard = indexShardList.get(randomIndexPosition);
+                TermQueryBuilder termQuery = new TermQueryBuilder("id", generateString(randomIntBetween(4, 50)));
+                BytesReference termBytes = null;
+                try {
+                    termBytes = XContentHelper.toXContent(termQuery, MediaTypeRegistry.JSON, false);
+                } catch (IOException e) {
+                    throw new RuntimeException(e);
+                }
+                Loader loader = new Loader(readerMap.get(indexShard), finalI);
+                try {
+                    cache.getOrCompute(entityMap.get(indexShard), loader, readerMap.get(indexShard), termBytes);
+                } catch (Exception e) {
+                    throw new RuntimeException(e);
+                }
+                latch.countDown();
+            });
+        }
+        latch.await();
+        for (int i = 0; i < numberOfIndices; i++) {
+            IndexShard indexShard = indexShardList.get(i);
+            IndicesService.IndexShardCacheEntity entity = entityMap.get(indexShard);
+            RequestCacheStats stats = entity.stats().stats();
+            assertTrue(stats.getMemorySizeInBytes() >= 0);
+            assertTrue(stats.getMissCount() >= 0);
+            assertTrue(stats.getEvictions() >= 0);
+        }
+        cache.invalidateAll();
+        for (int i = 0; i < numberOfIndices; i++) {
+            IndexShard indexShard = indexShardList.get(i);
+            IndicesService.IndexShardCacheEntity entity = entityMap.get(indexShard);
+            RequestCacheStats stats = entity.stats().stats();
+            assertEquals(0, stats.getMemorySizeInBytes());
+        }
 
+        for (int i = 0; i < numberOfIndices; i++) {
+            IndexShard indexShard = indexShardList.get(i);
+            readerMap.get(indexShard).close();
+            writerMap.get(indexShard).close();
+            writerMap.get(indexShard).getDirectory().close();
+        }
+        IOUtils.close(cache);
+        executorService.shutdownNow();
+    }
+
+    public void testDeleteAndCreateIndexShardOnSameNodeAndVerifyStats() throws Exception {
+        threadPool = getThreadPool();
+        String indexName = "test1";
+        IndicesService indicesService = getInstanceFromNode(IndicesService.class);
+        // Create a shard
+        IndexService indexService = createIndex(
+            indexName,
+            Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).build()
+        );
+        Index idx = resolveIndex(indexName);
+        ShardRouting shardRouting = indicesService.indexService(idx).getShard(0).routingEntry();
+        IndexShard indexShard = indexService.getShard(0);
+        Directory dir = newDirectory();
+        IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
+        writer.addDocument(newDoc(0, "foo"));
+        writer.addDocument(newDoc(1, "hack"));
+        DirectoryReader reader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), indexShard.shardId());
+        Loader loader = new Loader(reader, 0);
+
+        // Set clean interval to a high value as we will do it manually here.
+        IndicesRequestCache cache = getIndicesRequestCache(
+            Settings.builder()
+                .put(IndicesRequestCache.INDICES_REQUEST_CACHE_CLEANUP_INTERVAL_SETTING_KEY, TimeValue.timeValueMillis(100000))
+                .build()
+        );
+        IndicesService.IndexShardCacheEntity cacheEntity = new IndicesService.IndexShardCacheEntity(indexShard);
+        TermQueryBuilder termQuery = new TermQueryBuilder("id", "bar");
+        BytesReference termBytes = XContentHelper.toXContent(termQuery, MediaTypeRegistry.JSON, false);
+
+        // Cache some values for indexShard
+        BytesReference value = cache.getOrCompute(cacheEntity, loader, reader, getTermBytes());
+
+        // Verify response and stats.
+        assertEquals("foo", value.streamInput().readString());
+        RequestCacheStats stats = indexShard.requestCache().stats();
+        assertEquals("foo", value.streamInput().readString());
+        assertEquals(1, cache.count());
+        assertEquals(1, stats.getMissCount());
+        assertTrue(stats.getMemorySizeInBytes() > 0);
+
+        // Remove the shard making its cache entries stale
+        IOUtils.close(reader, writer, dir);
+        indexService.removeShard(0, "force");
+
+        // We again try to create a shard with same ShardId
+        ShardRouting newRouting = shardRouting;
+        String nodeId = newRouting.currentNodeId();
+        UnassignedInfo unassignedInfo = new UnassignedInfo(UnassignedInfo.Reason.INDEX_CREATED, "boom");
+        newRouting = newRouting.moveToUnassigned(unassignedInfo)
+            .updateUnassigned(unassignedInfo, RecoverySource.EmptyStoreRecoverySource.INSTANCE);
+        newRouting = ShardRoutingHelper.initialize(newRouting, nodeId);
+        final DiscoveryNode localNode = new DiscoveryNode("foo", buildNewFakeTransportAddress(), emptyMap(), emptySet(), Version.CURRENT);
+        indexShard = indexService.createShard(
+            newRouting,
+            s -> {},
+            RetentionLeaseSyncer.EMPTY,
+            SegmentReplicationCheckpointPublisher.EMPTY,
+            null,
+            null,
+            localNode,
+            null,
+            DiscoveryNodes.builder().add(localNode).build()
+        );
+
+        // Verify that the new shard requestStats entries are empty.
+        stats = indexShard.requestCache().stats();
+        assertEquals("foo", value.streamInput().readString());
+        assertEquals(1, cache.count()); // Still contains the old indexShard stale entry
+        assertEquals(0, stats.getMissCount());
+        assertTrue(stats.getMemorySizeInBytes() == 0);
+        IndexShardTestCase.updateRoutingEntry(indexShard, newRouting);
+
+        // Now we cache again with new IndexShard(same shardId as older one).
+        dir = newDirectory();
+        writer = new IndexWriter(dir, newIndexWriterConfig());
+        writer.addDocument(newDoc(0, "foo"));
+        writer.addDocument(newDoc(1, "hack"));
+        reader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(writer), indexShard.shardId());
+        loader = new Loader(reader, 0);
+        cacheEntity = new IndicesService.IndexShardCacheEntity(indexShard);
+        termQuery = new TermQueryBuilder("id", "bar");
+        termBytes = XContentHelper.toXContent(termQuery, MediaTypeRegistry.JSON, false);
+        value = cache.getOrCompute(cacheEntity, loader, reader, getTermBytes());
+
+        // Assert response and stats. We verify that cache now has 2 entries, one for older/removed shard and other
+        // for the current shard.
+        assertEquals("foo", value.streamInput().readString());
+        stats = indexShard.requestCache().stats();
+        assertEquals("foo", value.streamInput().readString());
+        assertEquals(2, cache.count()); // One entry for older shard and other for the current shard.
+        assertEquals(1, stats.getMissCount());
+        assertTrue(stats.getMemorySizeInBytes() > 0);
+
+        // Trigger clean up of cache.
+        cache.cacheCleanupManager.cleanCache();
+        // Verify that cache still has entries for current shard and only removed older shards entries.
+        assertEquals(1, cache.count());
+
+        // Now make current indexShard entries stale as well.
+        reader.close();
+        // Trigger clean up of cache and verify that cache has no entries now.
+        cache.cacheCleanupManager.cleanCache();
+        assertEquals(0, cache.count());
+
+        IOUtils.close(reader, writer, dir, cache);
+    }
+
+    public static String generateString(int length) {
+        String characters = "abcdefghijklmnopqrstuvwxyz";
+        StringBuilder sb = new StringBuilder(length);
+        for (int i = 0; i < length; i++) {
+            int index = randomInt(characters.length() - 1);
+            sb.append(characters.charAt(index));
+        }
+        return sb.toString();
     }
 
     private class TestBytesReference extends AbstractBytesReference {
diff --git a/server/src/test/java/org/opensearch/indices/IndicesServiceTests.java b/server/src/test/java/org/opensearch/indices/IndicesServiceTests.java
index 742dbdeba8c5b..6757dbc184961 100644
--- a/server/src/test/java/org/opensearch/indices/IndicesServiceTests.java
+++ b/server/src/test/java/org/opensearch/indices/IndicesServiceTests.java
@@ -622,6 +622,9 @@ public void testConflictingEngineFactories() {
 
     public void testClusterRemoteTranslogBufferIntervalDefault() {
         IndicesService indicesService = getIndicesService();
-        assertEquals(IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL, indicesService.getClusterRemoteTranslogBufferInterval());
+        assertEquals(
+            IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL,
+            indicesService.getRemoteStoreSettings().getClusterRemoteTranslogBufferInterval()
+        );
     }
 }
diff --git a/server/src/test/java/org/opensearch/indices/RemoteStoreSettingsDynamicUpdateTests.java b/server/src/test/java/org/opensearch/indices/RemoteStoreSettingsDynamicUpdateTests.java
new file mode 100644
index 0000000000000..f89fd3df6e340
--- /dev/null
+++ b/server/src/test/java/org/opensearch/indices/RemoteStoreSettingsDynamicUpdateTests.java
@@ -0,0 +1,119 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.indices;
+
+import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.TimeValue;
+import org.opensearch.test.OpenSearchTestCase;
+
+import static org.opensearch.indices.RemoteStoreSettings.CLUSTER_REMOTE_TRANSLOG_TRANSFER_TIMEOUT_SETTING;
+
+public class RemoteStoreSettingsDynamicUpdateTests extends OpenSearchTestCase {
+    private final ClusterSettings clusterSettings = new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS);
+    private final RemoteStoreSettings remoteStoreSettings = new RemoteStoreSettings(Settings.EMPTY, clusterSettings);
+
+    public void testSegmentMetadataRetention() {
+        // Default value
+        assertEquals(10, remoteStoreSettings.getMinRemoteSegmentMetadataFiles());
+
+        // Setting value < default (10)
+        clusterSettings.applySettings(
+            Settings.builder()
+                .put(RemoteStoreSettings.CLUSTER_REMOTE_INDEX_SEGMENT_METADATA_RETENTION_MAX_COUNT_SETTING.getKey(), 5)
+                .build()
+        );
+        assertEquals(5, remoteStoreSettings.getMinRemoteSegmentMetadataFiles());
+
+        // Setting min value
+        clusterSettings.applySettings(
+            Settings.builder()
+                .put(RemoteStoreSettings.CLUSTER_REMOTE_INDEX_SEGMENT_METADATA_RETENTION_MAX_COUNT_SETTING.getKey(), -1)
+                .build()
+        );
+        assertEquals(-1, remoteStoreSettings.getMinRemoteSegmentMetadataFiles());
+
+        // Setting value > default (10)
+        clusterSettings.applySettings(
+            Settings.builder()
+                .put(RemoteStoreSettings.CLUSTER_REMOTE_INDEX_SEGMENT_METADATA_RETENTION_MAX_COUNT_SETTING.getKey(), 15)
+                .build()
+        );
+        assertEquals(15, remoteStoreSettings.getMinRemoteSegmentMetadataFiles());
+
+        // Setting value to 0 should fail and retain the existing value
+        assertThrows(
+            IllegalArgumentException.class,
+            () -> clusterSettings.applySettings(
+                Settings.builder()
+                    .put(RemoteStoreSettings.CLUSTER_REMOTE_INDEX_SEGMENT_METADATA_RETENTION_MAX_COUNT_SETTING.getKey(), 0)
+                    .build()
+            )
+        );
+        assertEquals(15, remoteStoreSettings.getMinRemoteSegmentMetadataFiles());
+
+        // Setting value < -1 should fail and retain the existing value
+        assertThrows(
+            IllegalArgumentException.class,
+            () -> clusterSettings.applySettings(
+                Settings.builder()
+                    .put(RemoteStoreSettings.CLUSTER_REMOTE_INDEX_SEGMENT_METADATA_RETENTION_MAX_COUNT_SETTING.getKey(), -5)
+                    .build()
+            )
+        );
+        assertEquals(15, remoteStoreSettings.getMinRemoteSegmentMetadataFiles());
+    }
+
+    public void testClusterRemoteTranslogTransferTimeout() {
+        // Test default value
+        assertEquals(TimeValue.timeValueSeconds(30), remoteStoreSettings.getClusterRemoteTranslogTransferTimeout());
+
+        // Test override with valid value
+        clusterSettings.applySettings(Settings.builder().put(CLUSTER_REMOTE_TRANSLOG_TRANSFER_TIMEOUT_SETTING.getKey(), "40s").build());
+        assertEquals(TimeValue.timeValueSeconds(40), remoteStoreSettings.getClusterRemoteTranslogTransferTimeout());
+
+        // Test override with value less than minimum
+        assertThrows(
+            IllegalArgumentException.class,
+            () -> clusterSettings.applySettings(
+                Settings.builder().put(CLUSTER_REMOTE_TRANSLOG_TRANSFER_TIMEOUT_SETTING.getKey(), "10s").build()
+            )
+        );
+        assertEquals(TimeValue.timeValueSeconds(40), remoteStoreSettings.getClusterRemoteTranslogTransferTimeout());
+
+        // Test override with invalid time value
+        assertThrows(
+            IllegalArgumentException.class,
+            () -> clusterSettings.applySettings(
+                Settings.builder().put(CLUSTER_REMOTE_TRANSLOG_TRANSFER_TIMEOUT_SETTING.getKey(), "123").build()
+            )
+        );
+        assertEquals(TimeValue.timeValueSeconds(40), remoteStoreSettings.getClusterRemoteTranslogTransferTimeout());
+    }
+
+    public void testMaxRemoteReferencedTranslogFiles() {
+        // Test default value
+        assertEquals(1000, remoteStoreSettings.getMaxRemoteTranslogReaders());
+
+        // Test override with valid value
+        clusterSettings.applySettings(
+            Settings.builder().put(RemoteStoreSettings.CLUSTER_REMOTE_MAX_TRANSLOG_READERS.getKey(), "500").build()
+        );
+        assertEquals(500, remoteStoreSettings.getMaxRemoteTranslogReaders());
+
+        // Test override with value less than minimum
+        assertThrows(
+            IllegalArgumentException.class,
+            () -> clusterSettings.applySettings(
+                Settings.builder().put(RemoteStoreSettings.CLUSTER_REMOTE_MAX_TRANSLOG_READERS.getKey(), "99").build()
+            )
+        );
+        assertEquals(500, remoteStoreSettings.getMaxRemoteTranslogReaders());
+    }
+}
diff --git a/server/src/test/java/org/opensearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java b/server/src/test/java/org/opensearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java
index c455101ff4549..0490228a5cc16 100644
--- a/server/src/test/java/org/opensearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java
+++ b/server/src/test/java/org/opensearch/indices/cluster/AbstractIndicesClusterStateServiceTestCase.java
@@ -35,6 +35,7 @@
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.cluster.node.DiscoveryNodes;
 import org.opensearch.cluster.routing.IndexShardRoutingTable;
 import org.opensearch.cluster.routing.RoutingNode;
 import org.opensearch.cluster.routing.ShardRouting;
@@ -264,7 +265,8 @@ public MockIndexShard createShard(
             final RetentionLeaseSyncer retentionLeaseSyncer,
             final DiscoveryNode targetNode,
             final DiscoveryNode sourceNode,
-            final RemoteStoreStatsTrackerFactory remoteStoreStatsTrackerFactory
+            final RemoteStoreStatsTrackerFactory remoteStoreStatsTrackerFactory,
+            final DiscoveryNodes discoveryNodes
         ) throws IOException {
             failRandomly();
             RecoveryState recoveryState = new RecoveryState(shardRouting, targetNode, sourceNode);
@@ -387,7 +389,8 @@ public void updateShardState(
             BiConsumer<IndexShard, ActionListener<ResyncTask>> primaryReplicaSyncer,
             long applyingClusterStateVersion,
             Set<String> inSyncAllocationIds,
-            IndexShardRoutingTable routingTable
+            IndexShardRoutingTable routingTable,
+            DiscoveryNodes discoveryNodes
         ) throws IOException {
             failRandomly();
             assertThat(this.shardId(), equalTo(shardRouting.shardId()));
diff --git a/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java b/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java
index dc4dca80ea110..17bd821ed0c8c 100644
--- a/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java
+++ b/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java
@@ -105,6 +105,7 @@
 import org.opensearch.index.IndexService;
 import org.opensearch.index.mapper.MapperService;
 import org.opensearch.index.shard.IndexEventListener;
+import org.opensearch.indices.DefaultRemoteStoreSettings;
 import org.opensearch.indices.IndicesService;
 import org.opensearch.indices.ShardLimitValidator;
 import org.opensearch.indices.SystemIndices;
@@ -312,7 +313,8 @@ public IndexMetadata upgradeIndexMetadata(IndexMetadata indexMetadata, Version m
             xContentRegistry,
             systemIndices,
             true,
-            awarenessReplicaBalance
+            awarenessReplicaBalance,
+            DefaultRemoteStoreSettings.INSTANCE
         );
 
         transportCloseIndexAction = new TransportCloseIndexAction(
diff --git a/server/src/test/java/org/opensearch/indices/recovery/PeerRecoverySourceServiceTests.java b/server/src/test/java/org/opensearch/indices/recovery/PeerRecoverySourceServiceTests.java
index 4fbae4b0d53ca..ded174fb98eef 100644
--- a/server/src/test/java/org/opensearch/indices/recovery/PeerRecoverySourceServiceTests.java
+++ b/server/src/test/java/org/opensearch/indices/recovery/PeerRecoverySourceServiceTests.java
@@ -38,6 +38,7 @@
 import org.opensearch.index.seqno.SequenceNumbers;
 import org.opensearch.index.shard.IndexShard;
 import org.opensearch.index.shard.IndexShardTestCase;
+import org.opensearch.index.shard.IndexShardTestUtils;
 import org.opensearch.index.store.Store;
 import org.opensearch.indices.IndicesService;
 import org.opensearch.test.NodeRoles;
@@ -65,8 +66,8 @@ public void testDuplicateRecoveries() throws IOException {
         StartRecoveryRequest startRecoveryRequest = new StartRecoveryRequest(
             primary.shardId(),
             randomAlphaOfLength(10),
-            getFakeDiscoNode("source"),
-            getFakeDiscoNode("target"),
+            IndexShardTestUtils.getFakeDiscoNode("source"),
+            IndexShardTestUtils.getFakeDiscoNode("target"),
             Store.MetadataSnapshot.EMPTY,
             randomBoolean(),
             randomLong(),
diff --git a/server/src/test/java/org/opensearch/indices/recovery/PeerRecoveryTargetServiceTests.java b/server/src/test/java/org/opensearch/indices/recovery/PeerRecoveryTargetServiceTests.java
index 1e6cc43703672..a8e5a02011538 100644
--- a/server/src/test/java/org/opensearch/indices/recovery/PeerRecoveryTargetServiceTests.java
+++ b/server/src/test/java/org/opensearch/indices/recovery/PeerRecoveryTargetServiceTests.java
@@ -56,6 +56,7 @@
 import org.opensearch.index.seqno.SequenceNumbers;
 import org.opensearch.index.shard.IndexShard;
 import org.opensearch.index.shard.IndexShardTestCase;
+import org.opensearch.index.shard.IndexShardTestUtils;
 import org.opensearch.index.store.Store;
 import org.opensearch.index.store.StoreFileMetadata;
 import org.opensearch.index.translog.Translog;
@@ -92,8 +93,8 @@ public void testWriteFileChunksConcurrently() throws Exception {
             mdFiles.add(md);
         }
         final IndexShard targetShard = newShard(false);
-        final DiscoveryNode pNode = getFakeDiscoNode(sourceShard.routingEntry().currentNodeId());
-        final DiscoveryNode rNode = getFakeDiscoNode(targetShard.routingEntry().currentNodeId());
+        final DiscoveryNode pNode = IndexShardTestUtils.getFakeDiscoNode(sourceShard.routingEntry().currentNodeId());
+        final DiscoveryNode rNode = IndexShardTestUtils.getFakeDiscoNode(targetShard.routingEntry().currentNodeId());
         targetShard.markAsRecovering("test-peer-recovery", new RecoveryState(targetShard.routingEntry(), rNode, pNode));
         final RecoveryTarget recoveryTarget = new RecoveryTarget(targetShard, null, null, threadPool);
         final PlainActionFuture<Void> receiveFileInfoFuture = new PlainActionFuture<>();
diff --git a/server/src/test/java/org/opensearch/indices/recovery/RecoverySettingsDynamicUpdateTests.java b/server/src/test/java/org/opensearch/indices/recovery/RecoverySettingsDynamicUpdateTests.java
index 18e7dfb375132..2793d446d66c8 100644
--- a/server/src/test/java/org/opensearch/indices/recovery/RecoverySettingsDynamicUpdateTests.java
+++ b/server/src/test/java/org/opensearch/indices/recovery/RecoverySettingsDynamicUpdateTests.java
@@ -35,6 +35,8 @@
 import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
+import org.opensearch.core.common.unit.ByteSizeUnit;
+import org.opensearch.core.common.unit.ByteSizeValue;
 import org.opensearch.test.OpenSearchTestCase;
 
 import java.util.concurrent.TimeUnit;
@@ -47,7 +49,27 @@ public void testZeroBytesPerSecondIsNoRateLimit() {
         clusterSettings.applySettings(
             Settings.builder().put(RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey(), 0).build()
         );
-        assertEquals(null, recoverySettings.rateLimiter());
+        assertNull(recoverySettings.recoveryRateLimiter());
+        clusterSettings.applySettings(
+            Settings.builder().put(RecoverySettings.INDICES_REPLICATION_MAX_BYTES_PER_SEC_SETTING.getKey(), 0).build()
+        );
+        assertNull(recoverySettings.replicationRateLimiter());
+    }
+
+    public void testSetReplicationMaxBytesPerSec() {
+        assertEquals(40, (int) recoverySettings.replicationRateLimiter().getMBPerSec());
+        clusterSettings.applySettings(
+            Settings.builder()
+                .put(RecoverySettings.INDICES_RECOVERY_MAX_BYTES_PER_SEC_SETTING.getKey(), new ByteSizeValue(60, ByteSizeUnit.MB))
+                .build()
+        );
+        assertEquals(60, (int) recoverySettings.replicationRateLimiter().getMBPerSec());
+        clusterSettings.applySettings(
+            Settings.builder()
+                .put(RecoverySettings.INDICES_REPLICATION_MAX_BYTES_PER_SEC_SETTING.getKey(), new ByteSizeValue(80, ByteSizeUnit.MB))
+                .build()
+        );
+        assertEquals(80, (int) recoverySettings.replicationRateLimiter().getMBPerSec());
     }
 
     public void testRetryDelayStateSync() {
@@ -96,49 +118,4 @@ public void testInternalLongActionTimeout() {
         );
         assertEquals(new TimeValue(duration, timeUnit), recoverySettings.internalActionLongTimeout());
     }
-
-    public void testSegmentMetadataRetention() {
-        // Default value
-        assertEquals(10, recoverySettings.getMinRemoteSegmentMetadataFiles());
-
-        // Setting value < default (10)
-        clusterSettings.applySettings(
-            Settings.builder().put(RecoverySettings.CLUSTER_REMOTE_INDEX_SEGMENT_METADATA_RETENTION_MAX_COUNT_SETTING.getKey(), 5).build()
-        );
-        assertEquals(5, recoverySettings.getMinRemoteSegmentMetadataFiles());
-
-        // Setting min value
-        clusterSettings.applySettings(
-            Settings.builder().put(RecoverySettings.CLUSTER_REMOTE_INDEX_SEGMENT_METADATA_RETENTION_MAX_COUNT_SETTING.getKey(), -1).build()
-        );
-        assertEquals(-1, recoverySettings.getMinRemoteSegmentMetadataFiles());
-
-        // Setting value > default (10)
-        clusterSettings.applySettings(
-            Settings.builder().put(RecoverySettings.CLUSTER_REMOTE_INDEX_SEGMENT_METADATA_RETENTION_MAX_COUNT_SETTING.getKey(), 15).build()
-        );
-        assertEquals(15, recoverySettings.getMinRemoteSegmentMetadataFiles());
-
-        // Setting value to 0 should fail and retain the existing value
-        assertThrows(
-            IllegalArgumentException.class,
-            () -> clusterSettings.applySettings(
-                Settings.builder()
-                    .put(RecoverySettings.CLUSTER_REMOTE_INDEX_SEGMENT_METADATA_RETENTION_MAX_COUNT_SETTING.getKey(), 0)
-                    .build()
-            )
-        );
-        assertEquals(15, recoverySettings.getMinRemoteSegmentMetadataFiles());
-
-        // Setting value < -1 should fail and retain the existing value
-        assertThrows(
-            IllegalArgumentException.class,
-            () -> clusterSettings.applySettings(
-                Settings.builder()
-                    .put(RecoverySettings.CLUSTER_REMOTE_INDEX_SEGMENT_METADATA_RETENTION_MAX_COUNT_SETTING.getKey(), -5)
-                    .build()
-            )
-        );
-        assertEquals(15, recoverySettings.getMinRemoteSegmentMetadataFiles());
-    }
 }
diff --git a/server/src/test/java/org/opensearch/indices/replication/OngoingSegmentReplicationsTests.java b/server/src/test/java/org/opensearch/indices/replication/OngoingSegmentReplicationsTests.java
index 44e2653cf01da..eb27850000bdd 100644
--- a/server/src/test/java/org/opensearch/indices/replication/OngoingSegmentReplicationsTests.java
+++ b/server/src/test/java/org/opensearch/indices/replication/OngoingSegmentReplicationsTests.java
@@ -25,7 +25,6 @@
 import org.opensearch.indices.recovery.FileChunkWriter;
 import org.opensearch.indices.recovery.RecoverySettings;
 import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint;
-import org.opensearch.indices.replication.common.CopyState;
 import org.opensearch.indices.replication.common.ReplicationType;
 import org.opensearch.transport.TransportService;
 import org.junit.Assert;
@@ -106,25 +105,21 @@ public void testPrepareAndSendSegments() throws IOException {
         final FileChunkWriter segmentSegmentFileChunkWriter = (fileMetadata, position, content, lastChunk, totalTranslogOps, listener) -> {
             listener.onResponse(null);
         };
-        final CopyState copyState = replications.prepareForReplication(request, segmentSegmentFileChunkWriter);
-        assertTrue(replications.isInCopyStateMap(request.getCheckpoint()));
+        final SegmentReplicationSourceHandler handler = replications.prepareForReplication(request, segmentSegmentFileChunkWriter);
         assertEquals(1, replications.size());
-        assertEquals(1, copyState.refCount());
 
         getSegmentFilesRequest = new GetSegmentFilesRequest(
             1L,
             replica.routingEntry().allocationId().getId(),
             replicaDiscoveryNode,
-            new ArrayList<>(copyState.getMetadataMap().values()),
+            new ArrayList<>(handler.getCheckpoint().getMetadataMap().values()),
             testCheckpoint
         );
 
         replications.startSegmentCopy(getSegmentFilesRequest, new ActionListener<>() {
             @Override
             public void onResponse(GetSegmentFilesResponse getSegmentFilesResponse) {
-                assertEquals(copyState.getMetadataMap().size(), getSegmentFilesResponse.files.size());
-                assertEquals(0, copyState.refCount());
-                assertFalse(replications.isInCopyStateMap(request.getCheckpoint()));
+                assertEquals(handler.getCheckpoint().getMetadataMap().size(), getSegmentFilesResponse.files.size());
                 assertEquals(0, replications.size());
             }
 
@@ -148,14 +143,11 @@ public void testCancelReplication() throws IOException {
             // this shouldn't be called in this test.
             Assert.fail();
         };
-        final CopyState copyState = replications.prepareForReplication(request, segmentSegmentFileChunkWriter);
+        final SegmentReplicationSourceHandler handler = replications.prepareForReplication(request, segmentSegmentFileChunkWriter);
         assertEquals(1, replications.size());
-        assertEquals(1, replications.cachedCopyStateSize());
 
         replications.cancelReplication(primaryDiscoveryNode);
-        assertEquals(0, copyState.refCount());
         assertEquals(0, replications.size());
-        assertEquals(0, replications.cachedCopyStateSize());
     }
 
     public void testCancelReplication_AfterSendFilesStarts() throws IOException, InterruptedException {
@@ -174,14 +166,13 @@ public void testCancelReplication_AfterSendFilesStarts() throws IOException, Int
             // cancel the replication as soon as the writer starts sending files.
             replications.cancel(replica.routingEntry().allocationId().getId(), "Test");
         };
-        final CopyState copyState = replications.prepareForReplication(request, segmentSegmentFileChunkWriter);
+        final SegmentReplicationSourceHandler handler = replications.prepareForReplication(request, segmentSegmentFileChunkWriter);
         assertEquals(1, replications.size());
-        assertEquals(1, replications.cachedCopyStateSize());
         getSegmentFilesRequest = new GetSegmentFilesRequest(
             1L,
             replica.routingEntry().allocationId().getId(),
             replicaDiscoveryNode,
-            new ArrayList<>(copyState.getMetadataMap().values()),
+            new ArrayList<>(handler.getCheckpoint().getMetadataMap().values()),
             testCheckpoint
         );
         replications.startSegmentCopy(getSegmentFilesRequest, new ActionListener<>() {
@@ -193,9 +184,7 @@ public void onResponse(GetSegmentFilesResponse getSegmentFilesResponse) {
             @Override
             public void onFailure(Exception e) {
                 assertEquals(CancellableThreads.ExecutionCancelledException.class, e.getClass());
-                assertEquals(0, copyState.refCount());
                 assertEquals(0, replications.size());
-                assertEquals(0, replications.cachedCopyStateSize());
                 latch.countDown();
             }
         });
@@ -219,8 +208,7 @@ public void testMultipleReplicasUseSameCheckpoint() throws IOException {
             Assert.fail();
         };
 
-        final CopyState copyState = replications.prepareForReplication(request, segmentSegmentFileChunkWriter);
-        assertEquals(1, copyState.refCount());
+        final SegmentReplicationSourceHandler handler = replications.prepareForReplication(request, segmentSegmentFileChunkWriter);
 
         final CheckpointInfoRequest secondRequest = new CheckpointInfoRequest(
             1L,
@@ -230,15 +218,11 @@ public void testMultipleReplicasUseSameCheckpoint() throws IOException {
         );
         replications.prepareForReplication(secondRequest, segmentSegmentFileChunkWriter);
 
-        assertEquals(2, copyState.refCount());
         assertEquals(2, replications.size());
-        assertEquals(1, replications.cachedCopyStateSize());
 
         replications.cancelReplication(primaryDiscoveryNode);
         replications.cancelReplication(replicaDiscoveryNode);
-        assertEquals(0, copyState.refCount());
         assertEquals(0, replications.size());
-        assertEquals(0, replications.cachedCopyStateSize());
         closeShards(secondReplica);
     }
 
@@ -280,8 +264,7 @@ public void testShardAlreadyReplicatingToNode() throws IOException {
             listener.onResponse(null);
         };
         replications.prepareForReplication(request, segmentSegmentFileChunkWriter);
-        CopyState copyState = replications.prepareForReplication(request, segmentSegmentFileChunkWriter);
-        assertEquals(1, copyState.refCount());
+        final SegmentReplicationSourceHandler handler = replications.prepareForReplication(request, segmentSegmentFileChunkWriter);
     }
 
     public void testStartReplicationWithNoFilesToFetch() throws IOException {
@@ -296,10 +279,8 @@ public void testStartReplicationWithNoFilesToFetch() throws IOException {
         // mock the FileChunkWriter so we can assert its ever called.
         final FileChunkWriter segmentSegmentFileChunkWriter = mock(FileChunkWriter.class);
         // Prepare for replication step - and ensure copyState is added to cache.
-        final CopyState copyState = replications.prepareForReplication(request, segmentSegmentFileChunkWriter);
-        assertTrue(replications.isInCopyStateMap(request.getCheckpoint()));
+        final SegmentReplicationSourceHandler handler = replications.prepareForReplication(request, segmentSegmentFileChunkWriter);
         assertEquals(1, replications.size());
-        assertEquals(1, copyState.refCount());
 
         getSegmentFilesRequest = new GetSegmentFilesRequest(
             1L,
@@ -314,8 +295,6 @@ public void testStartReplicationWithNoFilesToFetch() throws IOException {
             @Override
             public void onResponse(GetSegmentFilesResponse getSegmentFilesResponse) {
                 assertEquals(Collections.emptyList(), getSegmentFilesResponse.files);
-                assertEquals(0, copyState.refCount());
-                assertFalse(replications.isInCopyStateMap(request.getCheckpoint()));
                 verifyNoInteractions(segmentSegmentFileChunkWriter);
             }
 
@@ -340,8 +319,7 @@ public void testCancelAllReplicationsForShard() throws IOException {
             testCheckpoint
         );
 
-        final CopyState copyState = replications.prepareForReplication(request, mock(FileChunkWriter.class));
-        assertEquals(1, copyState.refCount());
+        final SegmentReplicationSourceHandler handler = replications.prepareForReplication(request, mock(FileChunkWriter.class));
 
         final CheckpointInfoRequest secondRequest = new CheckpointInfoRequest(
             1L,
@@ -351,15 +329,11 @@ public void testCancelAllReplicationsForShard() throws IOException {
         );
         replications.prepareForReplication(secondRequest, mock(FileChunkWriter.class));
 
-        assertEquals(2, copyState.refCount());
         assertEquals(2, replications.size());
-        assertEquals(1, replications.cachedCopyStateSize());
 
         // cancel the primary's ongoing replications.
         replications.cancel(primary, "Test");
-        assertEquals(0, copyState.refCount());
         assertEquals(0, replications.size());
-        assertEquals(0, replications.cachedCopyStateSize());
         closeShards(replica_2);
     }
 
@@ -372,8 +346,7 @@ public void testCancelForMissingIds() throws IOException {
         final String replicaAllocationId = replica.routingEntry().allocationId().getId();
         final CheckpointInfoRequest request = new CheckpointInfoRequest(1L, replicaAllocationId, primaryDiscoveryNode, testCheckpoint);
 
-        final CopyState copyState = replications.prepareForReplication(request, mock(FileChunkWriter.class));
-        assertEquals(1, copyState.refCount());
+        final SegmentReplicationSourceHandler handler = replications.prepareForReplication(request, mock(FileChunkWriter.class));
 
         final String replica_2AllocationId = replica_2.routingEntry().allocationId().getId();
         final CheckpointInfoRequest secondRequest = new CheckpointInfoRequest(
@@ -384,23 +357,17 @@ public void testCancelForMissingIds() throws IOException {
         );
         replications.prepareForReplication(secondRequest, mock(FileChunkWriter.class));
 
-        assertEquals(2, copyState.refCount());
         assertEquals(2, replications.size());
         assertTrue(replications.getHandlers().containsKey(replicaAllocationId));
         assertTrue(replications.getHandlers().containsKey(replica_2AllocationId));
-        assertEquals(1, replications.cachedCopyStateSize());
 
         replications.clearOutOfSyncIds(primary.shardId(), Set.of(replica_2AllocationId));
-        assertEquals(1, copyState.refCount());
         assertEquals(1, replications.size());
         assertTrue(replications.getHandlers().containsKey(replica_2AllocationId));
-        assertEquals(1, replications.cachedCopyStateSize());
 
         // cancel the primary's ongoing replications.
         replications.clearOutOfSyncIds(primary.shardId(), Collections.emptySet());
-        assertEquals(0, copyState.refCount());
         assertEquals(0, replications.size());
-        assertEquals(0, replications.cachedCopyStateSize());
         closeShards(replica_2);
     }
 
@@ -409,11 +376,8 @@ public void testPrepareForReplicationAlreadyReplicating() throws IOException {
         final String replicaAllocationId = replica.routingEntry().allocationId().getId();
         final CheckpointInfoRequest request = new CheckpointInfoRequest(1L, replicaAllocationId, primaryDiscoveryNode, testCheckpoint);
 
-        final CopyState copyState = replications.prepareForReplication(request, mock(FileChunkWriter.class));
-
-        final SegmentReplicationSourceHandler handler = replications.getHandlers().get(replicaAllocationId);
-        assertEquals(handler.getCopyState(), copyState);
-        assertEquals(1, copyState.refCount());
+        final SegmentReplicationSourceHandler handler = replications.prepareForReplication(request, mock(FileChunkWriter.class));
+        assertEquals(handler, replications.getHandlers().get(replicaAllocationId));
 
         ReplicationCheckpoint secondCheckpoint = new ReplicationCheckpoint(
             testCheckpoint.getShardId(),
@@ -430,11 +394,10 @@ public void testPrepareForReplicationAlreadyReplicating() throws IOException {
             secondCheckpoint
         );
 
-        final CopyState secondCopyState = replications.prepareForReplication(secondRequest, mock(FileChunkWriter.class));
-        final SegmentReplicationSourceHandler secondHandler = replications.getHandlers().get(replicaAllocationId);
-        assertEquals(secondHandler.getCopyState(), secondCopyState);
-        assertEquals("New copy state is incref'd", 1, secondCopyState.refCount());
-        assertEquals("Old copy state is cleaned up", 0, copyState.refCount());
-
+        final SegmentReplicationSourceHandler secondHandler = replications.prepareForReplication(
+            secondRequest,
+            mock(FileChunkWriter.class)
+        );
+        assertEquals(secondHandler, replications.getHandlers().get(replicaAllocationId));
     }
 }
diff --git a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationSourceHandlerTests.java b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationSourceHandlerTests.java
index d586767290797..901dc28794cfc 100644
--- a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationSourceHandlerTests.java
+++ b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationSourceHandlerTests.java
@@ -68,18 +68,16 @@ public void testSendFiles() throws IOException {
         chunkWriter = (fileMetadata, position, content, lastChunk, totalTranslogOps, listener) -> listener.onResponse(null);
 
         final ReplicationCheckpoint latestReplicationCheckpoint = primary.getLatestReplicationCheckpoint();
-        final CopyState copyState = new CopyState(latestReplicationCheckpoint, primary);
         SegmentReplicationSourceHandler handler = new SegmentReplicationSourceHandler(
             localNode,
             chunkWriter,
-            threadPool,
-            copyState,
+            primary,
             replica.routingEntry().allocationId().getId(),
             5000,
             1
         );
 
-        final List<StoreFileMetadata> expectedFiles = List.copyOf(copyState.getMetadataMap().values());
+        final List<StoreFileMetadata> expectedFiles = List.copyOf(handler.getCheckpoint().getMetadataMap().values());
 
         final GetSegmentFilesRequest getSegmentFilesRequest = new GetSegmentFilesRequest(
             1L,
@@ -106,12 +104,10 @@ public void testSendFiles_emptyRequest() throws IOException {
         chunkWriter = mock(FileChunkWriter.class);
 
         final ReplicationCheckpoint latestReplicationCheckpoint = primary.getLatestReplicationCheckpoint();
-        final CopyState copyState = new CopyState(latestReplicationCheckpoint, primary);
         SegmentReplicationSourceHandler handler = new SegmentReplicationSourceHandler(
             localNode,
             chunkWriter,
-            threadPool,
-            copyState,
+            primary,
             replica.routingEntry().allocationId().getId(),
             5000,
             1
@@ -148,12 +144,11 @@ public void testSendFileFails() throws IOException {
         );
 
         final ReplicationCheckpoint latestReplicationCheckpoint = primary.getLatestReplicationCheckpoint();
-        final CopyState copyState = new CopyState(latestReplicationCheckpoint, primary);
+        final CopyState copyState = new CopyState(primary);
         SegmentReplicationSourceHandler handler = new SegmentReplicationSourceHandler(
             localNode,
             chunkWriter,
-            threadPool,
-            copyState,
+            primary,
             primary.routingEntry().allocationId().getId(),
             5000,
             1
@@ -180,19 +175,18 @@ public void onFailure(Exception e) {
                 assertEquals(e.getClass(), OpenSearchException.class);
             }
         });
-        copyState.decRef();
+        copyState.close();
     }
 
     public void testReplicationAlreadyRunning() throws IOException {
         chunkWriter = mock(FileChunkWriter.class);
 
         final ReplicationCheckpoint latestReplicationCheckpoint = primary.getLatestReplicationCheckpoint();
-        final CopyState copyState = new CopyState(latestReplicationCheckpoint, primary);
+        final CopyState copyState = new CopyState(primary);
         SegmentReplicationSourceHandler handler = new SegmentReplicationSourceHandler(
             localNode,
             chunkWriter,
-            threadPool,
-            copyState,
+            primary,
             replica.routingEntry().allocationId().getId(),
             5000,
             1
@@ -217,12 +211,10 @@ public void testCancelReplication() throws IOException, InterruptedException {
         chunkWriter = mock(FileChunkWriter.class);
 
         final ReplicationCheckpoint latestReplicationCheckpoint = primary.getLatestReplicationCheckpoint();
-        final CopyState copyState = new CopyState(latestReplicationCheckpoint, primary);
         SegmentReplicationSourceHandler handler = new SegmentReplicationSourceHandler(
             localNode,
             chunkWriter,
-            threadPool,
-            copyState,
+            primary,
             primary.routingEntry().allocationId().getId(),
             5000,
             1
diff --git a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java
index 3c72dda2d8b5d..1faaa16ce5628 100644
--- a/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java
+++ b/server/src/test/java/org/opensearch/indices/replication/SegmentReplicationTargetServiceTests.java
@@ -262,8 +262,12 @@ public void testAlreadyOnNewCheckpoint() {
     }
 
     @TestLogging(reason = "Getting trace logs from replication package", value = "org.opensearch.indices.replication:TRACE")
-    public void testShardAlreadyReplicating() {
+    public void testShardAlreadyReplicating() throws InterruptedException {
+        // in this case shard is already replicating and we receive an ahead checkpoint with same pterm.
+        // ongoing replication is not cancelled and new one does not start.
         CountDownLatch blockGetCheckpointMetadata = new CountDownLatch(1);
+        CountDownLatch continueGetCheckpointMetadata = new CountDownLatch(1);
+        CountDownLatch replicationCompleteLatch = new CountDownLatch(1);
         SegmentReplicationSource source = new TestReplicationSource() {
             @Override
             public void getCheckpointMetadata(
@@ -272,14 +276,13 @@ public void getCheckpointMetadata(
                 ActionListener<CheckpointInfoResponse> listener
             ) {
                 try {
-                    blockGetCheckpointMetadata.await();
-                    final CopyState copyState = new CopyState(
-                        ReplicationCheckpoint.empty(primaryShard.shardId(), primaryShard.getLatestReplicationCheckpoint().getCodec()),
-                        primaryShard
-                    );
-                    listener.onResponse(
-                        new CheckpointInfoResponse(copyState.getCheckpoint(), copyState.getMetadataMap(), copyState.getInfosBytes())
-                    );
+                    blockGetCheckpointMetadata.countDown();
+                    continueGetCheckpointMetadata.await();
+                    try (final CopyState copyState = new CopyState(primaryShard)) {
+                        listener.onResponse(
+                            new CheckpointInfoResponse(copyState.getCheckpoint(), copyState.getMetadataMap(), copyState.getInfosBytes())
+                        );
+                    }
                 } catch (InterruptedException | IOException e) {
                     throw new RuntimeException(e);
                 }
@@ -300,24 +303,73 @@ public void getSegmentFiles(
         final SegmentReplicationTarget target = spy(
             new SegmentReplicationTarget(
                 replicaShard,
-                primaryShard.getLatestReplicationCheckpoint(),
+                initialCheckpoint,
                 source,
-                mock(SegmentReplicationTargetService.SegmentReplicationListener.class)
+                new SegmentReplicationTargetService.SegmentReplicationListener() {
+                    @Override
+                    public void onReplicationDone(SegmentReplicationState state) {
+                        replicationCompleteLatch.countDown();
+                    }
+
+                    @Override
+                    public void onReplicationFailure(
+                        SegmentReplicationState state,
+                        ReplicationFailedException e,
+                        boolean sendShardFailure
+                    ) {
+                        Assert.fail("Replication should not fail");
+                    }
+                }
             )
         );
 
         final SegmentReplicationTargetService spy = spy(sut);
-        doReturn(false).when(spy).processLatestReceivedCheckpoint(eq(replicaShard), any());
         // Start first round of segment replication.
         spy.startReplication(target);
+        // wait until we are at getCheckpointMetadata stage
+        blockGetCheckpointMetadata.await(5, TimeUnit.MINUTES);
 
-        // Start second round of segment replication, this should fail to start as first round is still in-progress
-        spy.onNewCheckpoint(newPrimaryCheckpoint, replicaShard);
-        verify(spy, times(1)).processLatestReceivedCheckpoint(eq(replicaShard), any());
-        blockGetCheckpointMetadata.countDown();
+        // try and insert a new target directly - it should fail immediately and alert listener
+        spy.startReplication(
+            new SegmentReplicationTarget(
+                replicaShard,
+                aheadCheckpoint,
+                source,
+                new SegmentReplicationTargetService.SegmentReplicationListener() {
+                    @Override
+                    public void onReplicationDone(SegmentReplicationState state) {
+                        Assert.fail("Should not succeed");
+                    }
+
+                    @Override
+                    public void onReplicationFailure(
+                        SegmentReplicationState state,
+                        ReplicationFailedException e,
+                        boolean sendShardFailure
+                    ) {
+                        assertFalse(sendShardFailure);
+                        assertEquals("Shard " + replicaShard.shardId() + " is already replicating", e.getMessage());
+                    }
+                }
+            )
+        );
+
+        // Start second round of segment replication through onNewCheckpoint, this should fail to start as first round is still in-progress
+        // aheadCheckpoint is of same pterm but higher version
+        assertTrue(replicaShard.shouldProcessCheckpoint(aheadCheckpoint));
+        spy.onNewCheckpoint(aheadCheckpoint, replicaShard);
+        verify(spy, times(0)).processLatestReceivedCheckpoint(eq(replicaShard), any());
+        // start replication is not invoked with aheadCheckpoint
+        verify(spy, times(0)).startReplication(
+            eq(replicaShard),
+            eq(aheadCheckpoint),
+            any(SegmentReplicationTargetService.SegmentReplicationListener.class)
+        );
+        continueGetCheckpointMetadata.countDown();
+        replicationCompleteLatch.await(5, TimeUnit.MINUTES);
     }
 
-    public void testOnNewCheckpointFromNewPrimaryCancelOngoingReplication() throws InterruptedException {
+    public void testShardAlreadyReplicating_HigherPrimaryTermReceived() throws InterruptedException {
         // Create a spy of Target Service so that we can verify invocation of startReplication call with specific checkpoint on it.
         SegmentReplicationTargetService serviceSpy = spy(sut);
         doNothing().when(serviceSpy).updateVisibleCheckpoint(anyLong(), any());
diff --git a/server/src/test/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointActionTests.java b/server/src/test/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointActionTests.java
index 2cf006176022d..352f827c74cb2 100644
--- a/server/src/test/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointActionTests.java
+++ b/server/src/test/java/org/opensearch/indices/replication/checkpoint/PublishCheckpointActionTests.java
@@ -15,6 +15,7 @@
 import org.opensearch.action.support.replication.ReplicationMode;
 import org.opensearch.action.support.replication.TransportReplicationAction;
 import org.opensearch.cluster.action.shard.ShardStateAction;
+import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.util.io.IOUtils;
@@ -35,9 +36,12 @@
 import java.util.Collections;
 import java.util.concurrent.atomic.AtomicBoolean;
 
+import static org.opensearch.index.remote.RemoteStoreTestsHelper.createIndexSettings;
 import static org.opensearch.test.ClusterServiceUtils.createClusterService;
 import static org.hamcrest.Matchers.sameInstance;
+import static org.mockito.Mockito.any;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
@@ -128,7 +132,9 @@ public void testPublishCheckpointActionOnReplica() {
 
         final ShardId shardId = new ShardId(index, id);
         when(indexShard.shardId()).thenReturn(shardId);
-
+        when(indexShard.indexSettings()).thenReturn(
+            createIndexSettings(false, Settings.builder().put(IndexMetadata.INDEX_REPLICATION_TYPE_SETTING.getKey(), "SEGMENT").build())
+        );
         final SegmentReplicationTargetService mockTargetService = mock(SegmentReplicationTargetService.class);
 
         final PublishCheckpointAction action = new PublishCheckpointAction(
@@ -160,17 +166,46 @@ public void testPublishCheckpointActionOnReplica() {
 
     }
 
+    public void testPublishCheckpointActionOnDocrepReplicaDuringMigration() {
+        final IndicesService indicesService = mock(IndicesService.class);
+
+        final Index index = new Index("index", "uuid");
+        final IndexService indexService = mock(IndexService.class);
+        when(indicesService.indexServiceSafe(index)).thenReturn(indexService);
+        final int id = randomIntBetween(0, 4);
+        final IndexShard indexShard = mock(IndexShard.class);
+        when(indexService.getShard(id)).thenReturn(indexShard);
+
+        final ShardId shardId = new ShardId(index, id);
+        when(indexShard.shardId()).thenReturn(shardId);
+        when(indexShard.indexSettings()).thenReturn(createIndexSettings(false));
+        final SegmentReplicationTargetService mockTargetService = mock(SegmentReplicationTargetService.class);
+
+        final PublishCheckpointAction action = new PublishCheckpointAction(
+            Settings.EMPTY,
+            transportService,
+            clusterService,
+            indicesService,
+            threadPool,
+            shardStateAction,
+            new ActionFilters(Collections.emptySet()),
+            mockTargetService
+        );
+        // no interaction with SegmentReplicationTargetService object
+        verify(mockTargetService, never()).onNewCheckpoint(any(), any());
+    }
+
     public void testGetReplicationModeWithRemoteTranslog() {
         final PublishCheckpointAction action = createAction();
         final IndexShard indexShard = mock(IndexShard.class);
-        when(indexShard.isRemoteTranslogEnabled()).thenReturn(true);
+        when(indexShard.indexSettings()).thenReturn(createIndexSettings(true));
         assertEquals(ReplicationMode.FULL_REPLICATION, action.getReplicationMode(indexShard));
     }
 
     public void testGetReplicationModeWithLocalTranslog() {
         final PublishCheckpointAction action = createAction();
         final IndexShard indexShard = mock(IndexShard.class);
-        when(indexShard.isRemoteTranslogEnabled()).thenReturn(false);
+        when(indexShard.indexSettings()).thenReturn(createIndexSettings(false));
         assertEquals(ReplicationMode.FULL_REPLICATION, action.getReplicationMode(indexShard));
     }
 
diff --git a/server/src/test/java/org/opensearch/indices/replication/common/CopyStateTests.java b/server/src/test/java/org/opensearch/indices/replication/common/CopyStateTests.java
index df180a8ab1007..0b30486038e3a 100644
--- a/server/src/test/java/org/opensearch/indices/replication/common/CopyStateTests.java
+++ b/server/src/test/java/org/opensearch/indices/replication/common/CopyStateTests.java
@@ -18,7 +18,6 @@
 import org.opensearch.common.settings.Settings;
 import org.opensearch.core.index.shard.ShardId;
 import org.opensearch.env.Environment;
-import org.opensearch.index.codec.CodecService;
 import org.opensearch.index.shard.IndexShard;
 import org.opensearch.index.shard.IndexShardTestCase;
 import org.opensearch.index.store.Store;
@@ -54,13 +53,7 @@ public class CopyStateTests extends IndexShardTestCase {
 
     public void testCopyStateCreation() throws IOException {
         final IndexShard mockIndexShard = createMockIndexShard();
-        CopyState copyState = new CopyState(
-            ReplicationCheckpoint.empty(
-                mockIndexShard.shardId(),
-                new CodecService(null, mockIndexShard.indexSettings(), null).codec("default").getName()
-            ),
-            mockIndexShard
-        );
+        CopyState copyState = new CopyState(mockIndexShard);
         ReplicationCheckpoint checkpoint = copyState.getCheckpoint();
         assertEquals(TEST_SHARD_ID, checkpoint.getShardId());
         // version was never set so this should be zero
@@ -86,7 +79,9 @@ public static IndexShard createMockIndexShard() throws IOException {
             mockShard.getOperationPrimaryTerm(),
             0L,
             0L,
-            Codec.getDefault().getName()
+            0L,
+            Codec.getDefault().getName(),
+            SI_SNAPSHOT.asMap()
         );
         final Tuple<GatedCloseable<SegmentInfos>, ReplicationCheckpoint> gatedCloseableReplicationCheckpointTuple = new Tuple<>(
             new GatedCloseable<>(testSegmentInfos, () -> {}),
diff --git a/server/src/test/java/org/opensearch/indices/store/ShardAttributesTests.java b/server/src/test/java/org/opensearch/indices/store/ShardAttributesTests.java
index 7fa95fefe72fd..94834bab1d98b 100644
--- a/server/src/test/java/org/opensearch/indices/store/ShardAttributesTests.java
+++ b/server/src/test/java/org/opensearch/indices/store/ShardAttributesTests.java
@@ -28,13 +28,12 @@ public class ShardAttributesTests extends OpenSearchTestCase {
     String customDataPath = "/path/to/data";
 
     public void testShardAttributesConstructor() {
-        ShardAttributes attributes = new ShardAttributes(shardId, customDataPath);
-        assertEquals(attributes.getShardId(), shardId);
+        ShardAttributes attributes = new ShardAttributes(customDataPath);
         assertEquals(attributes.getCustomDataPath(), customDataPath);
     }
 
     public void testSerialization() throws IOException {
-        ShardAttributes attributes1 = new ShardAttributes(shardId, customDataPath);
+        ShardAttributes attributes1 = new ShardAttributes(customDataPath);
         ByteArrayOutputStream bytes = new ByteArrayOutputStream();
         StreamOutput output = new DataOutputStreamOutput(new DataOutputStream(bytes));
         attributes1.writeTo(output);
@@ -42,7 +41,6 @@ public void testSerialization() throws IOException {
         StreamInput input = new InputStreamStreamInput(new ByteArrayInputStream(bytes.toByteArray()));
         ShardAttributes attributes2 = new ShardAttributes(input);
         input.close();
-        assertEquals(attributes1.getShardId(), attributes2.getShardId());
         assertEquals(attributes1.getCustomDataPath(), attributes2.getCustomDataPath());
     }
 
diff --git a/server/src/test/java/org/opensearch/ingest/CompoundProcessorTests.java b/server/src/test/java/org/opensearch/ingest/CompoundProcessorTests.java
index 76301acac0c19..aad6063bd3f4d 100644
--- a/server/src/test/java/org/opensearch/ingest/CompoundProcessorTests.java
+++ b/server/src/test/java/org/opensearch/ingest/CompoundProcessorTests.java
@@ -37,16 +37,23 @@
 import org.opensearch.test.OpenSearchTestCase;
 import org.junit.Before;
 
+import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.Comparator;
 import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.BiConsumer;
 import java.util.function.Consumer;
 import java.util.function.LongSupplier;
 
 import static java.util.Collections.singletonList;
+import static org.opensearch.ingest.IngestDocumentPreparer.SHOULD_FAIL_KEY;
 import static org.hamcrest.CoreMatchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.is;
@@ -429,6 +436,211 @@ public String getType() {
         assertThat(ingestProcessorException.getHeader("pipeline_origin"), equalTo(Arrays.asList("2", "1")));
     }
 
+    public void testBatchExecute_happyCase() {
+        List<IngestDocumentWrapper> wrapperList = Arrays.asList(
+            IngestDocumentPreparer.createIngestDocumentWrapper(1),
+            IngestDocumentPreparer.createIngestDocumentWrapper(2),
+            IngestDocumentPreparer.createIngestDocumentWrapper(3)
+        );
+        TestProcessor firstProcessor = new TestProcessor(doc -> {});
+        TestProcessor secondProcessor = new TestProcessor(doc -> {});
+        LongSupplier relativeTimeProvider = mock(LongSupplier.class);
+        CompoundProcessor compoundProcessor = new CompoundProcessor(
+            false,
+            Arrays.asList(firstProcessor, secondProcessor),
+            null,
+            relativeTimeProvider
+        );
+
+        compoundProcessor.batchExecute(wrapperList, results -> {
+            assertEquals(firstProcessor.getInvokedCounter(), wrapperList.size());
+            assertEquals(secondProcessor.getInvokedCounter(), wrapperList.size());
+            assertEquals(results.size(), wrapperList.size());
+            OperationStats stats = compoundProcessor.getProcessorsWithMetrics().get(0).v2().createStats();
+            assertEquals(0, stats.getCurrent());
+            assertEquals(3, stats.getCount());
+            for (int i = 0; i < wrapperList.size(); ++i) {
+                assertEquals(wrapperList.get(i).getSlot(), results.get(i).getSlot());
+                assertEquals(wrapperList.get(i).getIngestDocument(), results.get(i).getIngestDocument());
+                assertEquals(wrapperList.get(i).getException(), results.get(i).getException());
+            }
+        });
+    }
+
+    public void testBatchExecute_documentToDrop() {
+        List<IngestDocumentWrapper> wrapperList = Arrays.asList(
+            IngestDocumentPreparer.createIngestDocumentWrapper(1),
+            IngestDocumentPreparer.createIngestDocumentWrapper(2, true),
+            IngestDocumentPreparer.createIngestDocumentWrapper(3)
+        );
+        TestProcessor firstProcessor = new TestProcessor("", "", "", doc -> {
+            if (doc.hasField(SHOULD_FAIL_KEY) && doc.getFieldValue(SHOULD_FAIL_KEY, Boolean.class)) {
+                return null;
+            }
+            return doc;
+        });
+        TestProcessor secondProcessor = new TestProcessor(doc -> {});
+        LongSupplier relativeTimeProvider = mock(LongSupplier.class);
+        CompoundProcessor compoundProcessor = new CompoundProcessor(
+            false,
+            Arrays.asList(firstProcessor, secondProcessor),
+            null,
+            relativeTimeProvider
+        );
+
+        AtomicInteger callCounter = new AtomicInteger();
+        List<IngestDocumentWrapper> totalResults = Collections.synchronizedList(new ArrayList<>());
+        compoundProcessor.batchExecute(wrapperList, results -> {
+            totalResults.addAll(results);
+            if (callCounter.addAndGet(results.size()) == 3) {
+                assertEquals(firstProcessor.getInvokedCounter(), wrapperList.size());
+                assertEquals(secondProcessor.getInvokedCounter(), wrapperList.size() - 1);
+                assertEquals(totalResults.size(), wrapperList.size());
+                OperationStats stats = compoundProcessor.getProcessorsWithMetrics().get(0).v2().createStats();
+                assertEquals(0, stats.getCurrent());
+                assertEquals(3, stats.getCount());
+                totalResults.sort(Comparator.comparingInt(IngestDocumentWrapper::getSlot));
+                for (int i = 0; i < wrapperList.size(); ++i) {
+                    assertEquals(wrapperList.get(i).getSlot(), totalResults.get(i).getSlot());
+                    if (2 == wrapperList.get(i).getSlot()) {
+                        assertNull(totalResults.get(i).getIngestDocument());
+                    } else {
+                        assertEquals(wrapperList.get(i).getIngestDocument(), totalResults.get(i).getIngestDocument());
+                    }
+                    assertEquals(wrapperList.get(i).getException(), totalResults.get(i).getException());
+                }
+            }
+        });
+    }
+
+    public void testBatchExecute_ignoreFailure() {
+        List<IngestDocumentWrapper> wrapperList = Arrays.asList(
+            IngestDocumentPreparer.createIngestDocumentWrapper(1),
+            IngestDocumentPreparer.createIngestDocumentWrapper(2, true),
+            IngestDocumentPreparer.createIngestDocumentWrapper(3, true)
+        );
+        TestProcessor firstProcessor = new TestProcessor(doc -> {
+            if (doc.hasField(SHOULD_FAIL_KEY) && doc.getFieldValue(SHOULD_FAIL_KEY, Boolean.class)) {
+                throw new RuntimeException("fail");
+            }
+        });
+        TestProcessor secondProcessor = new TestProcessor(doc -> {});
+        TestProcessor onFailureProcessor = new TestProcessor("id2", "on_failure", null, doc -> {});
+        LongSupplier relativeTimeProvider = mock(LongSupplier.class);
+        CompoundProcessor compoundProcessor = new CompoundProcessor(
+            true,
+            Arrays.asList(firstProcessor, secondProcessor),
+            singletonList(onFailureProcessor),
+            relativeTimeProvider
+        );
+
+        compoundProcessor.batchExecute(wrapperList, results -> {
+            assertEquals(firstProcessor.getInvokedCounter(), wrapperList.size());
+            assertEquals(secondProcessor.getInvokedCounter(), wrapperList.size());
+            assertEquals(0, onFailureProcessor.getInvokedCounter());
+            assertEquals(results.size(), wrapperList.size());
+            OperationStats stats = compoundProcessor.getProcessorsWithMetrics().get(0).v2().createStats();
+            assertEquals(0, stats.getCurrent());
+            assertEquals(3, stats.getCount());
+            for (int i = 0; i < wrapperList.size(); ++i) {
+                assertEquals(wrapperList.get(i).getSlot(), results.get(i).getSlot());
+                assertEquals(wrapperList.get(i).getIngestDocument(), results.get(i).getIngestDocument());
+                assertEquals(wrapperList.get(i).getException(), results.get(i).getException());
+            }
+        });
+    }
+
+    public void testBatchExecute_exception_no_onFailureProcessor() {
+        Set<Integer> failureSlot = new HashSet<>(Arrays.asList(2, 3));
+        List<IngestDocumentWrapper> wrapperList = Arrays.asList(
+            IngestDocumentPreparer.createIngestDocumentWrapper(1),
+            IngestDocumentPreparer.createIngestDocumentWrapper(2, true),
+            IngestDocumentPreparer.createIngestDocumentWrapper(3, true)
+        );
+        TestProcessor firstProcessor = new TestProcessor(doc -> {
+            if (doc.hasField(SHOULD_FAIL_KEY) && doc.getFieldValue(SHOULD_FAIL_KEY, Boolean.class)) {
+                throw new RuntimeException("fail");
+            }
+        });
+        TestProcessor secondProcessor = new TestProcessor(doc -> {});
+        LongSupplier relativeTimeProvider = mock(LongSupplier.class);
+        CompoundProcessor compoundProcessor = new CompoundProcessor(
+            false,
+            Arrays.asList(firstProcessor, secondProcessor),
+            Collections.emptyList(),
+            relativeTimeProvider
+        );
+
+        AtomicInteger callCounter = new AtomicInteger();
+        List<IngestDocumentWrapper> totalResults = Collections.synchronizedList(new ArrayList<>());
+        compoundProcessor.batchExecute(wrapperList, results -> {
+            totalResults.addAll(results);
+            if (callCounter.incrementAndGet() == 3) {
+                assertEquals(wrapperList.size(), firstProcessor.getInvokedCounter());
+                assertEquals(1, secondProcessor.getInvokedCounter());
+                assertEquals(totalResults.size(), wrapperList.size());
+                OperationStats stats = compoundProcessor.getProcessorsWithMetrics().get(0).v2().createStats();
+                assertEquals(0, stats.getCurrent());
+                assertEquals(3, stats.getCount());
+                assertEquals(2, stats.getFailedCount());
+                totalResults.sort(Comparator.comparingInt(IngestDocumentWrapper::getSlot));
+                for (int i = 0; i < wrapperList.size(); ++i) {
+                    assertEquals(wrapperList.get(i).getSlot(), totalResults.get(i).getSlot());
+                    if (failureSlot.contains(wrapperList.get(i).getSlot())) {
+                        assertNotNull(totalResults.get(i).getException());
+                    } else {
+                        assertEquals(wrapperList.get(i).getIngestDocument(), totalResults.get(i).getIngestDocument());
+                        assertEquals(wrapperList.get(i).getException(), totalResults.get(i).getException());
+                    }
+                }
+            }
+        });
+    }
+
+    public void testBatchExecute_exception_with_onFailureProcessor() {
+        List<IngestDocumentWrapper> wrapperList = Arrays.asList(
+            IngestDocumentPreparer.createIngestDocumentWrapper(1),
+            IngestDocumentPreparer.createIngestDocumentWrapper(2, true),
+            IngestDocumentPreparer.createIngestDocumentWrapper(3, true)
+        );
+        TestProcessor firstProcessor = new TestProcessor(doc -> {
+            if (doc.hasField(SHOULD_FAIL_KEY) && doc.getFieldValue(SHOULD_FAIL_KEY, Boolean.class)) {
+                throw new RuntimeException("fail");
+            }
+        });
+        TestProcessor secondProcessor = new TestProcessor(doc -> {});
+        TestProcessor onFailureProcessor = new TestProcessor("id2", "on_failure", null, doc -> {});
+        LongSupplier relativeTimeProvider = mock(LongSupplier.class);
+        CompoundProcessor compoundProcessor = new CompoundProcessor(
+            false,
+            Arrays.asList(firstProcessor, secondProcessor),
+            singletonList(onFailureProcessor),
+            relativeTimeProvider
+        );
+
+        AtomicInteger callCounter = new AtomicInteger();
+        List<IngestDocumentWrapper> totalResults = Collections.synchronizedList(new ArrayList<>());
+        compoundProcessor.batchExecute(wrapperList, results -> {
+            totalResults.addAll(results);
+            if (callCounter.incrementAndGet() == 3) {
+                assertEquals(wrapperList.size(), firstProcessor.getInvokedCounter());
+                assertEquals(1, secondProcessor.getInvokedCounter());
+                assertEquals(2, onFailureProcessor.getInvokedCounter());
+                assertEquals(totalResults.size(), wrapperList.size());
+                OperationStats stats = compoundProcessor.getProcessorsWithMetrics().get(0).v2().createStats();
+                assertEquals(0, stats.getCurrent());
+                assertEquals(3, stats.getCount());
+                assertEquals(2, stats.getFailedCount());
+                totalResults.sort(Comparator.comparingInt(IngestDocumentWrapper::getSlot));
+                for (int i = 0; i < wrapperList.size(); ++i) {
+                    assertEquals(wrapperList.get(i).getSlot(), totalResults.get(i).getSlot());
+                    assertEquals(wrapperList.get(i).getIngestDocument(), totalResults.get(i).getIngestDocument());
+                    assertNull(totalResults.get(i).getException());
+                }
+            }
+        });
+    }
+
     private void assertStats(CompoundProcessor compoundProcessor, long count, long failed, long time) {
         assertStats(0, compoundProcessor, 0L, count, failed, time);
     }
diff --git a/server/src/test/java/org/opensearch/ingest/IngestDocumentPreparer.java b/server/src/test/java/org/opensearch/ingest/IngestDocumentPreparer.java
new file mode 100644
index 0000000000000..a02595df5589d
--- /dev/null
+++ b/server/src/test/java/org/opensearch/ingest/IngestDocumentPreparer.java
@@ -0,0 +1,32 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.ingest;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class IngestDocumentPreparer {
+    public static final String SHOULD_FAIL_KEY = "shouldFail";
+
+    public static IngestDocument createIngestDocument(boolean shouldFail) {
+        Map<String, Object> source = new HashMap<>();
+        if (shouldFail) {
+            source.put(SHOULD_FAIL_KEY, true);
+        }
+        return new IngestDocument(source, new HashMap<>());
+    }
+
+    public static IngestDocumentWrapper createIngestDocumentWrapper(int slot) {
+        return createIngestDocumentWrapper(slot, false);
+    }
+
+    public static IngestDocumentWrapper createIngestDocumentWrapper(int slot, boolean shouldFail) {
+        return new IngestDocumentWrapper(slot, createIngestDocument(shouldFail), null);
+    }
+}
diff --git a/server/src/test/java/org/opensearch/ingest/IngestDocumentWrapperTests.java b/server/src/test/java/org/opensearch/ingest/IngestDocumentWrapperTests.java
new file mode 100644
index 0000000000000..9d09cd80abd05
--- /dev/null
+++ b/server/src/test/java/org/opensearch/ingest/IngestDocumentWrapperTests.java
@@ -0,0 +1,46 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.ingest;
+
+import org.opensearch.index.VersionType;
+import org.opensearch.test.OpenSearchTestCase;
+import org.junit.Before;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class IngestDocumentWrapperTests extends OpenSearchTestCase {
+
+    private IngestDocument ingestDocument;
+
+    private static final String INDEX = "index";
+    private static final String ID = "id";
+    private static final String ROUTING = "routing";
+    private static final Long VERSION = 1L;
+    private static final VersionType VERSION_TYPE = VersionType.INTERNAL;
+    private static final String DOCUMENT_KEY = "foo";
+    private static final String DOCUMENT_VALUE = "bar";
+    private static final int SLOT = 12;
+
+    @Before
+    public void setup() throws Exception {
+        super.setUp();
+        Map<String, Object> document = new HashMap<>();
+        document.put(DOCUMENT_KEY, DOCUMENT_VALUE);
+        ingestDocument = new IngestDocument(INDEX, ID, ROUTING, VERSION, VERSION_TYPE, document);
+    }
+
+    public void testIngestDocumentWrapper() {
+        Exception ex = new RuntimeException("runtime exception");
+        IngestDocumentWrapper wrapper = new IngestDocumentWrapper(SLOT, ingestDocument, ex);
+        assertEquals(wrapper.getSlot(), SLOT);
+        assertEquals(wrapper.getException(), ex);
+        assertEquals(wrapper.getIngestDocument(), ingestDocument);
+    }
+}
diff --git a/server/src/test/java/org/opensearch/ingest/IngestServiceTests.java b/server/src/test/java/org/opensearch/ingest/IngestServiceTests.java
index 2edfe87387c92..6d216370bae9a 100644
--- a/server/src/test/java/org/opensearch/ingest/IngestServiceTests.java
+++ b/server/src/test/java/org/opensearch/ingest/IngestServiceTests.java
@@ -116,6 +116,7 @@
 import static org.mockito.Mockito.doAnswer;
 import static org.mockito.Mockito.doThrow;
 import static org.mockito.Mockito.eq;
+import static org.mockito.Mockito.lenient;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.times;
@@ -132,6 +133,7 @@ public Map<String, Processor.Factory> getProcessors(Processor.Parameters paramet
     };
 
     private ThreadPool threadPool;
+    private BulkRequest mockBulkRequest;
 
     @Before
     public void setup() {
@@ -139,6 +141,8 @@ public void setup() {
         ExecutorService executorService = OpenSearchExecutors.newDirectExecutorService();
         when(threadPool.generic()).thenReturn(executorService);
         when(threadPool.executor(anyString())).thenReturn(executorService);
+        mockBulkRequest = mock(BulkRequest.class);
+        lenient().when(mockBulkRequest.batchSize()).thenReturn(1);
     }
 
     public void testIngestPlugin() {
@@ -210,7 +214,8 @@ public void testExecuteIndexPipelineDoesNotExist() {
             failureHandler,
             completionHandler,
             indexReq -> {},
-            Names.WRITE
+            Names.WRITE,
+            new BulkRequest()
         );
 
         assertTrue(failure.get());
@@ -761,7 +766,8 @@ public String getType() {
             failureHandler,
             completionHandler,
             indexReq -> {},
-            Names.WRITE
+            Names.WRITE,
+            bulkRequest
         );
 
         assertTrue(failure.get());
@@ -807,7 +813,8 @@ public void testExecuteBulkPipelineDoesNotExist() {
             failureHandler,
             completionHandler,
             indexReq -> {},
-            Names.WRITE
+            Names.WRITE,
+            bulkRequest
         );
         verify(failureHandler, times(1)).accept(
             argThat((Integer item) -> item == 2),
@@ -843,7 +850,8 @@ public void testExecuteSuccess() {
             failureHandler,
             completionHandler,
             indexReq -> {},
-            Names.WRITE
+            Names.WRITE,
+            mockBulkRequest
         );
         verify(failureHandler, never()).accept(any(), any());
         verify(completionHandler, times(1)).accept(Thread.currentThread(), null);
@@ -874,7 +882,8 @@ public void testExecuteEmptyPipeline() throws Exception {
             failureHandler,
             completionHandler,
             indexReq -> {},
-            Names.WRITE
+            Names.WRITE,
+            mockBulkRequest
         );
         verify(failureHandler, never()).accept(any(), any());
         verify(completionHandler, times(1)).accept(Thread.currentThread(), null);
@@ -933,7 +942,8 @@ public void testExecutePropagateAllMetadataUpdates() throws Exception {
             failureHandler,
             completionHandler,
             indexReq -> {},
-            Names.WRITE
+            Names.WRITE,
+            mockBulkRequest
         );
         verify(processor).execute(any(), any());
         verify(failureHandler, never()).accept(any(), any());
@@ -977,7 +987,8 @@ public void testExecuteFailure() throws Exception {
             failureHandler,
             completionHandler,
             indexReq -> {},
-            Names.WRITE
+            Names.WRITE,
+            mockBulkRequest
         );
         verify(processor).execute(eqIndexTypeId(indexRequest.version(), indexRequest.versionType(), emptyMap()), any());
         verify(failureHandler, times(1)).accept(eq(0), any(RuntimeException.class));
@@ -1035,7 +1046,8 @@ public void testExecuteSuccessWithOnFailure() throws Exception {
             failureHandler,
             completionHandler,
             indexReq -> {},
-            Names.WRITE
+            Names.WRITE,
+            mockBulkRequest
         );
         verify(failureHandler, never()).accept(eq(0), any(IngestProcessorException.class));
         verify(completionHandler, times(1)).accept(Thread.currentThread(), null);
@@ -1084,7 +1096,8 @@ public void testExecuteFailureWithNestedOnFailure() throws Exception {
             failureHandler,
             completionHandler,
             indexReq -> {},
-            Names.WRITE
+            Names.WRITE,
+            mockBulkRequest
         );
         verify(processor).execute(eqIndexTypeId(indexRequest.version(), indexRequest.versionType(), emptyMap()), any());
         verify(failureHandler, times(1)).accept(eq(0), any(RuntimeException.class));
@@ -1146,7 +1159,8 @@ public void testBulkRequestExecutionWithFailures() throws Exception {
             requestItemErrorHandler,
             completionHandler,
             indexReq -> {},
-            Names.WRITE
+            Names.WRITE,
+            bulkRequest
         );
 
         verify(requestItemErrorHandler, times(numIndexRequests)).accept(anyInt(), argThat(o -> o.getCause().equals(error)));
@@ -1204,7 +1218,8 @@ public void testBulkRequestExecution() throws Exception {
             requestItemErrorHandler,
             completionHandler,
             indexReq -> {},
-            Names.WRITE
+            Names.WRITE,
+            bulkRequest
         );
 
         verify(requestItemErrorHandler, never()).accept(any(), any());
@@ -1272,7 +1287,8 @@ public void testStats() throws Exception {
             failureHandler,
             completionHandler,
             indexReq -> {},
-            Names.WRITE
+            Names.WRITE,
+            mockBulkRequest
         );
         final IngestStats afterFirstRequestStats = ingestService.stats();
         assertThat(afterFirstRequestStats.getPipelineStats().size(), equalTo(2));
@@ -1296,7 +1312,8 @@ public void testStats() throws Exception {
             failureHandler,
             completionHandler,
             indexReq -> {},
-            Names.WRITE
+            Names.WRITE,
+            mockBulkRequest
         );
         final IngestStats afterSecondRequestStats = ingestService.stats();
         assertThat(afterSecondRequestStats.getPipelineStats().size(), equalTo(2));
@@ -1325,7 +1342,8 @@ public void testStats() throws Exception {
             failureHandler,
             completionHandler,
             indexReq -> {},
-            Names.WRITE
+            Names.WRITE,
+            mockBulkRequest
         );
         final IngestStats afterThirdRequestStats = ingestService.stats();
         assertThat(afterThirdRequestStats.getPipelineStats().size(), equalTo(2));
@@ -1358,7 +1376,8 @@ public void testStats() throws Exception {
             failureHandler,
             completionHandler,
             indexReq -> {},
-            Names.WRITE
+            Names.WRITE,
+            mockBulkRequest
         );
         final IngestStats afterForthRequestStats = ingestService.stats();
         assertThat(afterForthRequestStats.getPipelineStats().size(), equalTo(2));
@@ -1456,7 +1475,8 @@ public String getDescription() {
             failureHandler,
             completionHandler,
             dropHandler,
-            Names.WRITE
+            Names.WRITE,
+            bulkRequest
         );
         verify(failureHandler, never()).accept(any(), any());
         verify(completionHandler, times(1)).accept(Thread.currentThread(), null);
@@ -1543,7 +1563,8 @@ public void testCBORParsing() throws Exception {
                 (integer, e) -> {},
                 (thread, e) -> {},
                 indexReq -> {},
-                Names.WRITE
+                Names.WRITE,
+                mockBulkRequest
             );
         }
 
@@ -1672,6 +1693,283 @@ public void testResolveRequestOrDefaultPipelineAndFinalPipeline() {
         }
     }
 
+    public void testExecuteBulkRequestInBatch() {
+        CompoundProcessor mockCompoundProcessor = mockCompoundProcessor();
+        IngestService ingestService = createWithProcessors(
+            Collections.singletonMap("mock", (factories, tag, description, config) -> mockCompoundProcessor)
+        );
+        createPipeline("_id", ingestService);
+        BulkRequest bulkRequest = new BulkRequest();
+        IndexRequest indexRequest1 = new IndexRequest("_index").id("_id1").source(emptyMap()).setPipeline("_id").setFinalPipeline("_none");
+        bulkRequest.add(indexRequest1);
+        IndexRequest indexRequest2 = new IndexRequest("_index").id("_id2").source(emptyMap()).setPipeline("_id").setFinalPipeline("_none");
+        bulkRequest.add(indexRequest2);
+        IndexRequest indexRequest3 = new IndexRequest("_index").id("_id3").source(emptyMap()).setPipeline("_none").setFinalPipeline("_id");
+        bulkRequest.add(indexRequest3);
+        IndexRequest indexRequest4 = new IndexRequest("_index").id("_id4").source(emptyMap()).setPipeline("_id").setFinalPipeline("_none");
+        bulkRequest.add(indexRequest4);
+        bulkRequest.batchSize(2);
+        @SuppressWarnings("unchecked")
+        final BiConsumer<Integer, Exception> failureHandler = mock(BiConsumer.class);
+        @SuppressWarnings("unchecked")
+        final BiConsumer<Thread, Exception> completionHandler = mock(BiConsumer.class);
+        ingestService.executeBulkRequest(
+            4,
+            bulkRequest.requests(),
+            failureHandler,
+            completionHandler,
+            indexReq -> {},
+            Names.WRITE,
+            bulkRequest
+        );
+        verify(failureHandler, never()).accept(any(), any());
+        verify(completionHandler, times(1)).accept(Thread.currentThread(), null);
+        verify(mockCompoundProcessor, times(2)).batchExecute(any(), any());
+        verify(mockCompoundProcessor, never()).execute(any(), any());
+    }
+
+    public void testExecuteBulkRequestInBatchWithDefaultAndFinalPipeline() {
+        CompoundProcessor mockCompoundProcessor = mockCompoundProcessor();
+        IngestService ingestService = createWithProcessors(
+            Collections.singletonMap("mock", (factories, tag, description, config) -> mockCompoundProcessor)
+        );
+        ClusterState clusterState = createPipeline("_id", ingestService);
+        createPipeline("_final", ingestService, clusterState);
+        BulkRequest bulkRequest = new BulkRequest();
+        IndexRequest indexRequest1 = new IndexRequest("_index").id("_id1").source(emptyMap()).setPipeline("_id").setFinalPipeline("_final");
+        bulkRequest.add(indexRequest1);
+        IndexRequest indexRequest2 = new IndexRequest("_index").id("_id2").source(emptyMap()).setPipeline("_id").setFinalPipeline("_final");
+        bulkRequest.add(indexRequest2);
+        IndexRequest indexRequest3 = new IndexRequest("_index").id("_id3").source(emptyMap()).setPipeline("_id").setFinalPipeline("_final");
+        bulkRequest.add(indexRequest3);
+        IndexRequest indexRequest4 = new IndexRequest("_index").id("_id4").source(emptyMap()).setPipeline("_id").setFinalPipeline("_final");
+        bulkRequest.add(indexRequest4);
+        bulkRequest.batchSize(2);
+        @SuppressWarnings("unchecked")
+        final BiConsumer<Integer, Exception> failureHandler = mock(BiConsumer.class);
+        @SuppressWarnings("unchecked")
+        final BiConsumer<Thread, Exception> completionHandler = mock(BiConsumer.class);
+        ingestService.executeBulkRequest(
+            4,
+            bulkRequest.requests(),
+            failureHandler,
+            completionHandler,
+            indexReq -> {},
+            Names.WRITE,
+            bulkRequest
+        );
+        verify(failureHandler, never()).accept(any(), any());
+        verify(completionHandler, times(1)).accept(Thread.currentThread(), null);
+        verify(mockCompoundProcessor, times(4)).batchExecute(any(), any());
+        verify(mockCompoundProcessor, never()).execute(any(), any());
+    }
+
+    public void testExecuteBulkRequestInBatchFallbackWithOneDocument() {
+        CompoundProcessor mockCompoundProcessor = mockCompoundProcessor();
+        IngestService ingestService = createWithProcessors(
+            Collections.singletonMap("mock", (factories, tag, description, config) -> mockCompoundProcessor)
+        );
+        createPipeline("_id", ingestService);
+        BulkRequest bulkRequest = new BulkRequest();
+        IndexRequest indexRequest1 = new IndexRequest("_index").id("_id1").source(emptyMap()).setPipeline("_id").setFinalPipeline("_none");
+        bulkRequest.add(indexRequest1);
+        bulkRequest.batchSize(2);
+        @SuppressWarnings("unchecked")
+        final BiConsumer<Integer, Exception> failureHandler = mock(BiConsumer.class);
+        @SuppressWarnings("unchecked")
+        final BiConsumer<Thread, Exception> completionHandler = mock(BiConsumer.class);
+        ingestService.executeBulkRequest(
+            1,
+            bulkRequest.requests(),
+            failureHandler,
+            completionHandler,
+            indexReq -> {},
+            Names.WRITE,
+            bulkRequest
+        );
+        verify(failureHandler, never()).accept(any(), any());
+        verify(completionHandler, times(1)).accept(Thread.currentThread(), null);
+        verify(mockCompoundProcessor, never()).batchExecute(any(), any());
+        verify(mockCompoundProcessor, times(1)).execute(any(), any());
+    }
+
+    public void testExecuteBulkRequestInBatchNoValidPipeline() {
+        CompoundProcessor mockCompoundProcessor = mockCompoundProcessor();
+        IngestService ingestService = createWithProcessors(
+            Collections.singletonMap("mock", (factories, tag, description, config) -> mockCompoundProcessor)
+        );
+        createPipeline("_id", ingestService);
+        BulkRequest bulkRequest = new BulkRequest();
+        // will not be handled as not valid document type
+        IndexRequest indexRequest1 = new IndexRequest("_index").id("_id1")
+            .source(emptyMap())
+            .setPipeline("_none")
+            .setFinalPipeline("_none");
+        bulkRequest.add(indexRequest1);
+        IndexRequest indexRequest2 = new IndexRequest("_index").id("_id2")
+            .source(emptyMap())
+            .setPipeline("_none")
+            .setFinalPipeline("_none");
+        bulkRequest.add(indexRequest2);
+        bulkRequest.batchSize(2);
+        @SuppressWarnings("unchecked")
+        final BiConsumer<Integer, Exception> failureHandler = mock(BiConsumer.class);
+        @SuppressWarnings("unchecked")
+        final BiConsumer<Thread, Exception> completionHandler = mock(BiConsumer.class);
+        ingestService.executeBulkRequest(
+            2,
+            bulkRequest.requests(),
+            failureHandler,
+            completionHandler,
+            indexReq -> {},
+            Names.WRITE,
+            bulkRequest
+        );
+        verify(failureHandler, never()).accept(any(), any());
+        verify(completionHandler, times(1)).accept(Thread.currentThread(), null);
+        verify(mockCompoundProcessor, never()).batchExecute(any(), any());
+        verify(mockCompoundProcessor, never()).execute(any(), any());
+    }
+
+    public void testExecuteBulkRequestInBatchNoValidDocument() {
+        CompoundProcessor mockCompoundProcessor = mockCompoundProcessor();
+        IngestService ingestService = createWithProcessors(
+            Collections.singletonMap("mock", (factories, tag, description, config) -> mockCompoundProcessor)
+        );
+        createPipeline("_id", ingestService);
+        BulkRequest bulkRequest = new BulkRequest();
+        // will not be handled as not valid document type
+        bulkRequest.add(new DeleteRequest("_index", "_id"));
+        bulkRequest.add(new DeleteRequest("_index", "_id"));
+        bulkRequest.batchSize(2);
+        @SuppressWarnings("unchecked")
+        final BiConsumer<Integer, Exception> failureHandler = mock(BiConsumer.class);
+        @SuppressWarnings("unchecked")
+        final BiConsumer<Thread, Exception> completionHandler = mock(BiConsumer.class);
+        ingestService.executeBulkRequest(
+            2,
+            bulkRequest.requests(),
+            failureHandler,
+            completionHandler,
+            indexReq -> {},
+            Names.WRITE,
+            bulkRequest
+        );
+        verify(failureHandler, never()).accept(any(), any());
+        verify(completionHandler, times(1)).accept(Thread.currentThread(), null);
+        verify(mockCompoundProcessor, never()).batchExecute(any(), any());
+        verify(mockCompoundProcessor, never()).execute(any(), any());
+    }
+
+    public void testExecuteBulkRequestInBatchWithException() {
+        CompoundProcessor mockCompoundProcessor = mockCompoundProcessor();
+        IngestService ingestService = createWithProcessors(
+            Collections.singletonMap("mock", (factories, tag, description, config) -> mockCompoundProcessor)
+        );
+        doThrow(new RuntimeException()).when(mockCompoundProcessor).batchExecute(any(), any());
+        createPipeline("_id", ingestService);
+        BulkRequest bulkRequest = new BulkRequest();
+        // will not be handled as not valid document type
+        IndexRequest indexRequest1 = new IndexRequest("_index").id("_id1").source(emptyMap()).setPipeline("_id").setFinalPipeline("_none");
+        bulkRequest.add(indexRequest1);
+        IndexRequest indexRequest2 = new IndexRequest("_index").id("_id2").source(emptyMap()).setPipeline("_id").setFinalPipeline("_none");
+        bulkRequest.add(indexRequest2);
+        bulkRequest.batchSize(2);
+        @SuppressWarnings("unchecked")
+        final BiConsumer<Integer, Exception> failureHandler = mock(BiConsumer.class);
+        @SuppressWarnings("unchecked")
+        final BiConsumer<Thread, Exception> completionHandler = mock(BiConsumer.class);
+        ingestService.executeBulkRequest(
+            2,
+            bulkRequest.requests(),
+            failureHandler,
+            completionHandler,
+            indexReq -> {},
+            Names.WRITE,
+            bulkRequest
+        );
+        verify(failureHandler, times(2)).accept(any(), any());
+        verify(completionHandler, times(1)).accept(Thread.currentThread(), null);
+        verify(mockCompoundProcessor, times(1)).batchExecute(any(), any());
+        verify(mockCompoundProcessor, never()).execute(any(), any());
+    }
+
+    public void testExecuteBulkRequestInBatchWithExceptionInCallback() {
+        CompoundProcessor mockCompoundProcessor = mockCompoundProcessor();
+        IngestService ingestService = createWithProcessors(
+            Collections.singletonMap("mock", (factories, tag, description, config) -> mockCompoundProcessor)
+        );
+        createPipeline("_id", ingestService);
+        BulkRequest bulkRequest = new BulkRequest();
+        // will not be handled as not valid document type
+        IndexRequest indexRequest1 = new IndexRequest("_index").id("_id1").source(emptyMap()).setPipeline("_id").setFinalPipeline("_none");
+        bulkRequest.add(indexRequest1);
+        IndexRequest indexRequest2 = new IndexRequest("_index").id("_id2").source(emptyMap()).setPipeline("_id").setFinalPipeline("_none");
+        bulkRequest.add(indexRequest2);
+        bulkRequest.batchSize(2);
+
+        List<IngestDocumentWrapper> results = Arrays.asList(
+            new IngestDocumentWrapper(0, IngestService.toIngestDocument(indexRequest1), null),
+            new IngestDocumentWrapper(1, null, new RuntimeException())
+        );
+        doAnswer(args -> {
+            @SuppressWarnings("unchecked")
+            Consumer<List<IngestDocumentWrapper>> handler = (Consumer) args.getArguments()[1];
+            handler.accept(results);
+            return null;
+        }).when(mockCompoundProcessor).batchExecute(any(), any());
+
+        @SuppressWarnings("unchecked")
+        final BiConsumer<Integer, Exception> failureHandler = mock(BiConsumer.class);
+        @SuppressWarnings("unchecked")
+        final BiConsumer<Thread, Exception> completionHandler = mock(BiConsumer.class);
+        ingestService.executeBulkRequest(
+            2,
+            bulkRequest.requests(),
+            failureHandler,
+            completionHandler,
+            indexReq -> {},
+            Names.WRITE,
+            bulkRequest
+        );
+        verify(failureHandler, times(1)).accept(any(), any());
+        verify(completionHandler, times(1)).accept(Thread.currentThread(), null);
+        verify(mockCompoundProcessor, times(1)).batchExecute(any(), any());
+        verify(mockCompoundProcessor, never()).execute(any(), any());
+    }
+
+    public void testPrepareBatches_same_index_pipeline() {
+        IngestService.IndexRequestWrapper wrapper1 = createIndexRequestWrapper("index1", Collections.singletonList("p1"));
+        IngestService.IndexRequestWrapper wrapper2 = createIndexRequestWrapper("index1", Collections.singletonList("p1"));
+        IngestService.IndexRequestWrapper wrapper3 = createIndexRequestWrapper("index1", Collections.singletonList("p1"));
+        IngestService.IndexRequestWrapper wrapper4 = createIndexRequestWrapper("index1", Collections.singletonList("p1"));
+        List<List<IngestService.IndexRequestWrapper>> batches = IngestService.prepareBatches(
+            2,
+            Arrays.asList(wrapper1, wrapper2, wrapper3, wrapper4)
+        );
+        assertEquals(2, batches.size());
+        for (int i = 0; i < 2; ++i) {
+            assertEquals(2, batches.get(i).size());
+        }
+    }
+
+    public void testPrepareBatches_different_index_pipeline() {
+        IngestService.IndexRequestWrapper wrapper1 = createIndexRequestWrapper("index1", Collections.singletonList("p1"));
+        IngestService.IndexRequestWrapper wrapper2 = createIndexRequestWrapper("index2", Collections.singletonList("p1"));
+        IngestService.IndexRequestWrapper wrapper3 = createIndexRequestWrapper("index1", Arrays.asList("p1", "p2"));
+        IngestService.IndexRequestWrapper wrapper4 = createIndexRequestWrapper("index1", Collections.singletonList("p2"));
+        List<List<IngestService.IndexRequestWrapper>> batches = IngestService.prepareBatches(
+            2,
+            Arrays.asList(wrapper1, wrapper2, wrapper3, wrapper4)
+        );
+        assertEquals(4, batches.size());
+    }
+
+    private IngestService.IndexRequestWrapper createIndexRequestWrapper(String index, List<String> pipelines) {
+        IndexRequest indexRequest = new IndexRequest(index);
+        return new IngestService.IndexRequestWrapper(0, indexRequest, pipelines, true);
+    }
+
     private IngestDocument eqIndexTypeId(final Map<String, Object> source) {
         return argThat(new IngestDocumentMatcher("_index", "_type", "_id", -3L, VersionType.INTERNAL, source));
     }
@@ -1718,6 +2016,13 @@ private CompoundProcessor mockCompoundProcessor() {
             handler.accept((IngestDocument) args.getArguments()[0], null);
             return null;
         }).when(processor).execute(any(), any());
+
+        doAnswer(args -> {
+            @SuppressWarnings("unchecked")
+            Consumer<List<IngestDocumentWrapper>> handler = (Consumer) args.getArguments()[1];
+            handler.accept((List<IngestDocumentWrapper>) args.getArguments()[0]);
+            return null;
+        }).when(processor).batchExecute(any(), any());
         return processor;
     }
 
@@ -1757,4 +2062,24 @@ private void assertStats(OperationStats stats, long count, long failed, long tim
     private OperationStats getPipelineStats(List<IngestStats.PipelineStat> pipelineStats, String id) {
         return pipelineStats.stream().filter(p1 -> p1.getPipelineId().equals(id)).findFirst().map(p2 -> p2.getStats()).orElse(null);
     }
+
+    private ClusterState createPipeline(String pipeline, IngestService ingestService) {
+        return createPipeline(pipeline, ingestService, null);
+    }
+
+    private ClusterState createPipeline(String pipeline, IngestService ingestService, ClusterState previousState) {
+        PutPipelineRequest putRequest = new PutPipelineRequest(
+            pipeline,
+            new BytesArray("{\"processors\": [{\"mock\" : {}}]}"),
+            MediaTypeRegistry.JSON
+        );
+        ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build(); // Start empty
+        if (previousState != null) {
+            clusterState = previousState;
+        }
+        ClusterState previousClusterState = clusterState;
+        clusterState = IngestService.innerPut(putRequest, clusterState);
+        ingestService.applyClusterState(new ClusterChangedEvent("", clusterState, previousClusterState));
+        return clusterState;
+    }
 }
diff --git a/server/src/test/java/org/opensearch/ingest/ProcessorTests.java b/server/src/test/java/org/opensearch/ingest/ProcessorTests.java
new file mode 100644
index 0000000000000..d6ef3be73adb8
--- /dev/null
+++ b/server/src/test/java/org/opensearch/ingest/ProcessorTests.java
@@ -0,0 +1,74 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.ingest;
+
+import org.opensearch.test.OpenSearchTestCase;
+import org.junit.Before;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
+import static org.opensearch.ingest.IngestDocumentPreparer.SHOULD_FAIL_KEY;
+
+public class ProcessorTests extends OpenSearchTestCase {
+    private Processor processor;
+    private static final String FIELD_KEY = "result";
+    private static final String FIELD_VALUE_PROCESSED = "processed";
+
+    @Before
+    public void setup() {}
+
+    public void test_batchExecute_success() {
+        processor = new FakeProcessor("type", "tag", "description", doc -> { doc.setFieldValue(FIELD_KEY, FIELD_VALUE_PROCESSED); });
+        List<IngestDocumentWrapper> wrapperList = Arrays.asList(
+            IngestDocumentPreparer.createIngestDocumentWrapper(1),
+            IngestDocumentPreparer.createIngestDocumentWrapper(2),
+            IngestDocumentPreparer.createIngestDocumentWrapper(3)
+        );
+        processor.batchExecute(wrapperList, results -> {
+            assertEquals(3, results.size());
+            for (IngestDocumentWrapper wrapper : results) {
+                assertNull(wrapper.getException());
+                assertEquals(FIELD_VALUE_PROCESSED, wrapper.getIngestDocument().getFieldValue(FIELD_KEY, String.class));
+            }
+        });
+    }
+
+    public void test_batchExecute_empty() {
+        processor = new FakeProcessor("type", "tag", "description", doc -> { doc.setFieldValue(FIELD_KEY, FIELD_VALUE_PROCESSED); });
+        processor.batchExecute(Collections.emptyList(), results -> { assertEquals(0, results.size()); });
+    }
+
+    public void test_batchExecute_exception() {
+        processor = new FakeProcessor("type", "tag", "description", doc -> {
+            if (doc.hasField(SHOULD_FAIL_KEY) && doc.getFieldValue(SHOULD_FAIL_KEY, Boolean.class)) {
+                throw new RuntimeException("fail");
+            }
+            doc.setFieldValue(FIELD_KEY, FIELD_VALUE_PROCESSED);
+        });
+        List<IngestDocumentWrapper> wrapperList = Arrays.asList(
+            IngestDocumentPreparer.createIngestDocumentWrapper(1),
+            IngestDocumentPreparer.createIngestDocumentWrapper(2, true),
+            IngestDocumentPreparer.createIngestDocumentWrapper(3)
+        );
+        processor.batchExecute(wrapperList, results -> {
+            assertEquals(3, results.size());
+            for (IngestDocumentWrapper wrapper : results) {
+                if (wrapper.getSlot() == 2) {
+                    assertNotNull(wrapper.getException());
+                    assertNull(wrapper.getIngestDocument());
+                } else {
+                    assertNull(wrapper.getException());
+                    assertEquals(FIELD_VALUE_PROCESSED, wrapper.getIngestDocument().getFieldValue(FIELD_KEY, String.class));
+                }
+            }
+        });
+    }
+}
diff --git a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryHelperTests.java b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryHelperTests.java
index 57c126b85ff70..29ffb94ce8bf4 100644
--- a/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryHelperTests.java
+++ b/server/src/test/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryHelperTests.java
@@ -43,6 +43,8 @@
 import java.util.List;
 import java.util.Map;
 
+import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.SEGMENTS;
+import static org.opensearch.index.remote.RemoteStoreEnums.DataType.LOCK_FILES;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.greaterThan;
 import static org.hamcrest.Matchers.is;
@@ -56,14 +58,16 @@ protected Collection<Class<? extends Plugin>> getPlugins() {
     }
 
     protected String[] getLockFilesInRemoteStore(String remoteStoreIndex, String remoteStoreRepository) throws IOException {
-        String indexUUID = client().admin()
-            .indices()
-            .prepareGetSettings(remoteStoreIndex)
-            .get()
-            .getSetting(remoteStoreIndex, IndexMetadata.SETTING_INDEX_UUID);
         final RepositoriesService repositoriesService = getInstanceFromNode(RepositoriesService.class);
         final BlobStoreRepository remoteStorerepository = (BlobStoreRepository) repositoriesService.repository(remoteStoreRepository);
-        BlobPath shardLevelBlobPath = remoteStorerepository.basePath().add(indexUUID).add("0").add("segments").add("lock_files");
+        BlobPath shardLevelBlobPath = getShardLevelBlobPath(
+            client(),
+            remoteStoreIndex,
+            remoteStorerepository.basePath(),
+            "0",
+            SEGMENTS,
+            LOCK_FILES
+        );
         BlobContainer blobContainer = remoteStorerepository.blobStore().blobContainer(shardLevelBlobPath);
         try (RemoteBufferedOutputDirectory lockDirectory = new RemoteBufferedOutputDirectory(blobContainer)) {
             return Arrays.stream(lockDirectory.listAll())
diff --git a/server/src/test/java/org/opensearch/rest/BaseRestHandlerTests.java b/server/src/test/java/org/opensearch/rest/BaseRestHandlerTests.java
index ce929e64d8960..45653e9d8e4d6 100644
--- a/server/src/test/java/org/opensearch/rest/BaseRestHandlerTests.java
+++ b/server/src/test/java/org/opensearch/rest/BaseRestHandlerTests.java
@@ -35,10 +35,6 @@
 import org.opensearch.client.node.NodeClient;
 import org.opensearch.common.Table;
 import org.opensearch.common.settings.Settings;
-import org.opensearch.common.xcontent.json.JsonXContent;
-import org.opensearch.core.common.bytes.BytesArray;
-import org.opensearch.core.xcontent.MediaTypeRegistry;
-import org.opensearch.core.xcontent.XContentBuilder;
 import org.opensearch.rest.RestHandler.ReplacedRoute;
 import org.opensearch.rest.RestHandler.Route;
 import org.opensearch.rest.RestRequest.Method;
@@ -281,81 +277,6 @@ public String getName() {
         assertTrue(executed.get());
     }
 
-    public void testConsumedBody() throws Exception {
-        final AtomicBoolean executed = new AtomicBoolean();
-        final BaseRestHandler handler = new BaseRestHandler() {
-            @Override
-            protected RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
-                request.content();
-                return channel -> executed.set(true);
-            }
-
-            @Override
-            public String getName() {
-                return "test_consumed_body";
-            }
-        };
-
-        try (XContentBuilder builder = JsonXContent.contentBuilder().startObject().endObject()) {
-            final RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withContent(
-                new BytesArray(builder.toString()),
-                MediaTypeRegistry.JSON
-            ).build();
-            final RestChannel channel = new FakeRestChannel(request, randomBoolean(), 1);
-            handler.handleRequest(request, channel, mockClient);
-            assertTrue(executed.get());
-        }
-    }
-
-    public void testUnconsumedNoBody() throws Exception {
-        final AtomicBoolean executed = new AtomicBoolean();
-        final BaseRestHandler handler = new BaseRestHandler() {
-            @Override
-            protected RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
-                return channel -> executed.set(true);
-            }
-
-            @Override
-            public String getName() {
-                return "test_unconsumed_body";
-            }
-        };
-
-        final RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).build();
-        final RestChannel channel = new FakeRestChannel(request, randomBoolean(), 1);
-        handler.handleRequest(request, channel, mockClient);
-        assertTrue(executed.get());
-    }
-
-    public void testUnconsumedBody() throws IOException {
-        final AtomicBoolean executed = new AtomicBoolean();
-        final BaseRestHandler handler = new BaseRestHandler() {
-            @Override
-            protected RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
-                return channel -> executed.set(true);
-            }
-
-            @Override
-            public String getName() {
-                return "test_unconsumed_body";
-            }
-        };
-
-        try (XContentBuilder builder = JsonXContent.contentBuilder().startObject().endObject()) {
-            final RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withContent(
-                new BytesArray(builder.toString()),
-                MediaTypeRegistry.JSON
-            ).build();
-            final RestChannel channel = new FakeRestChannel(request, randomBoolean(), 1);
-            final IllegalArgumentException e = expectThrows(
-                IllegalArgumentException.class,
-                () -> handler.handleRequest(request, channel, mockClient)
-            );
-            assertThat(e, hasToString(containsString("request [GET /] does not support having a body")));
-            assertFalse(executed.get());
-        }
-    }
-
     public void testReplaceRoutesMethod() throws Exception {
         List<Route> routes = Arrays.asList(new Route(Method.GET, "/path/test"), new Route(Method.PUT, "/path2/test"));
         List<ReplacedRoute> replacedRoutes = RestHandler.replaceRoutes(routes, "/prefix", "/deprecatedPrefix");
diff --git a/server/src/test/java/org/opensearch/rest/action/cat/RestShardsActionTests.java b/server/src/test/java/org/opensearch/rest/action/cat/RestShardsActionTests.java
index fa13ec2036797..883df7da5d717 100644
--- a/server/src/test/java/org/opensearch/rest/action/cat/RestShardsActionTests.java
+++ b/server/src/test/java/org/opensearch/rest/action/cat/RestShardsActionTests.java
@@ -125,7 +125,7 @@ public void testBuildTable() {
         assertThat(headers.get(6).value, equalTo("ip"));
         assertThat(headers.get(7).value, equalTo("id"));
         assertThat(headers.get(8).value, equalTo("node"));
-        assertThat(headers.get(78).value, equalTo("docs.deleted"));
+        assertThat(headers.get(79).value, equalTo("docs.deleted"));
 
         final List<List<Table.Cell>> rows = table.getRows();
         assertThat(rows.size(), equalTo(numShards));
@@ -141,9 +141,9 @@ public void testBuildTable() {
             assertThat(row.get(4).value, equalTo(shardStats.getStats().getDocs().getCount()));
             assertThat(row.get(6).value, equalTo(localNode.getHostAddress()));
             assertThat(row.get(7).value, equalTo(localNode.getId()));
-            assertThat(row.get(76).value, equalTo(shardStats.getDataPath()));
-            assertThat(row.get(77).value, equalTo(shardStats.getStatePath()));
-            assertThat(row.get(78).value, equalTo(shardStats.getStats().getDocs().getDeleted()));
+            assertThat(row.get(77).value, equalTo(shardStats.getDataPath()));
+            assertThat(row.get(78).value, equalTo(shardStats.getStatePath()));
+            assertThat(row.get(79).value, equalTo(shardStats.getStats().getDocs().getDeleted()));
         }
     }
 }
diff --git a/server/src/test/java/org/opensearch/search/DefaultSearchContextTests.java b/server/src/test/java/org/opensearch/search/DefaultSearchContextTests.java
index 3793249d569f0..a1a808c9faa9b 100644
--- a/server/src/test/java/org/opensearch/search/DefaultSearchContextTests.java
+++ b/server/src/test/java/org/opensearch/search/DefaultSearchContextTests.java
@@ -93,6 +93,7 @@
 import java.util.function.Function;
 import java.util.function.Supplier;
 
+import static org.opensearch.index.IndexSettings.INDEX_SEARCH_THROTTLED;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
 import static org.mockito.Mockito.any;
@@ -168,6 +169,7 @@ public void testPreProcess() throws Exception {
         IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY);
         when(indexService.getIndexSettings()).thenReturn(indexSettings);
         when(mapperService.getIndexSettings()).thenReturn(indexSettings);
+        when(indexShard.indexSettings()).thenReturn(indexSettings);
 
         BigArrays bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService());
 
@@ -486,6 +488,14 @@ public void testClearQueryCancellationsOnClose() throws IOException {
         when(indexService.newQueryShardContext(eq(shardId.id()), any(), any(), nullable(String.class), anyBoolean())).thenReturn(
             queryShardContext
         );
+        Settings settings = Settings.builder()
+            .put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT)
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)
+            .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 2)
+            .build();
+        IndexMetadata indexMetadata = IndexMetadata.builder("index").settings(settings).build();
+        IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY);
+        when(indexShard.indexSettings()).thenReturn(indexSettings);
 
         BigArrays bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService());
 
@@ -551,7 +561,7 @@ protected Engine.Searcher acquireSearcherInternal(String source) {
         }
     }
 
-    public void testSearchPathEvaluationUsingSortField() throws Exception {
+    public void testSearchPathEvaluation() throws Exception {
         ShardSearchRequest shardSearchRequest = mock(ShardSearchRequest.class);
         when(shardSearchRequest.searchType()).thenReturn(SearchType.DEFAULT);
         ShardId shardId = new ShardId("index", UUID.randomUUID().toString(), 1);
@@ -578,9 +588,24 @@ public void testSearchPathEvaluationUsingSortField() throws Exception {
         IndexMetadata indexMetadata = IndexMetadata.builder("index").settings(settings).build();
         IndexSettings indexSettings = new IndexSettings(indexMetadata, Settings.EMPTY);
         when(indexService.getIndexSettings()).thenReturn(indexSettings);
+        when(indexShard.indexSettings()).thenReturn(indexSettings);
 
         BigArrays bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), new NoneCircuitBreakerService());
 
+        IndexShard systemIndexShard = mock(IndexShard.class);
+        when(systemIndexShard.getQueryCachingPolicy()).thenReturn(queryCachingPolicy);
+        when(systemIndexShard.getThreadPool()).thenReturn(threadPool);
+        when(systemIndexShard.isSystem()).thenReturn(true);
+
+        IndexShard throttledIndexShard = mock(IndexShard.class);
+        when(throttledIndexShard.getQueryCachingPolicy()).thenReturn(queryCachingPolicy);
+        when(throttledIndexShard.getThreadPool()).thenReturn(threadPool);
+        IndexSettings throttledIndexSettings = new IndexSettings(
+            indexMetadata,
+            Settings.builder().put(INDEX_SEARCH_THROTTLED.getKey(), true).build()
+        );
+        when(throttledIndexShard.indexSettings()).thenReturn(throttledIndexSettings);
+
         try (Directory dir = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random(), dir)) {
 
             final Supplier<Engine.SearcherSupplier> searcherSupplier = () -> new Engine.SearcherSupplier(Function.identity()) {
@@ -697,6 +722,62 @@ protected Engine.Searcher acquireSearcherInternal(String source) {
             }
             assertThrows(SetOnce.AlreadySetException.class, context::evaluateRequestShouldUseConcurrentSearch);
 
+            // Case 4: With a system index concurrent segment search is not used
+            readerContext = new ReaderContext(
+                newContextId(),
+                indexService,
+                systemIndexShard,
+                searcherSupplier.get(),
+                randomNonNegativeLong(),
+                false
+            );
+            context = new DefaultSearchContext(
+                readerContext,
+                shardSearchRequest,
+                target,
+                null,
+                bigArrays,
+                null,
+                null,
+                null,
+                false,
+                Version.CURRENT,
+                false,
+                executor,
+                null
+            );
+            context.evaluateRequestShouldUseConcurrentSearch();
+            assertFalse(context.shouldUseConcurrentSearch());
+            assertThrows(SetOnce.AlreadySetException.class, context::evaluateRequestShouldUseConcurrentSearch);
+
+            // Case 5: When search is throttled concurrent segment search is not used
+            readerContext = new ReaderContext(
+                newContextId(),
+                indexService,
+                throttledIndexShard,
+                searcherSupplier.get(),
+                randomNonNegativeLong(),
+                false
+            );
+            context = new DefaultSearchContext(
+                readerContext,
+                shardSearchRequest,
+                target,
+                null,
+                bigArrays,
+                null,
+                null,
+                null,
+                false,
+                Version.CURRENT,
+                false,
+                executor,
+                null
+            );
+            context.evaluateRequestShouldUseConcurrentSearch();
+            assertFalse(context.shouldUseConcurrentSearch());
+            assertThrows(SetOnce.AlreadySetException.class, context::evaluateRequestShouldUseConcurrentSearch);
+
             // shutdown the threadpool
             threadPool.shutdown();
         }
diff --git a/server/src/test/java/org/opensearch/search/SearchServiceTests.java b/server/src/test/java/org/opensearch/search/SearchServiceTests.java
index d502bab5918a8..7207df7d4d13d 100644
--- a/server/src/test/java/org/opensearch/search/SearchServiceTests.java
+++ b/server/src/test/java/org/opensearch/search/SearchServiceTests.java
@@ -71,6 +71,7 @@
 import org.opensearch.index.IndexService;
 import org.opensearch.index.IndexSettings;
 import org.opensearch.index.engine.Engine;
+import org.opensearch.index.mapper.DerivedFieldType;
 import org.opensearch.index.query.AbstractQueryBuilder;
 import org.opensearch.index.query.MatchAllQueryBuilder;
 import org.opensearch.index.query.MatchNoneQueryBuilder;
@@ -547,6 +548,49 @@ public void testMaxDocvalueFieldsSearch() throws IOException {
         }
     }
 
+    public void testDerivedFieldsSearch() throws IOException {
+        createIndex("index");
+        final SearchService service = getInstanceFromNode(SearchService.class);
+        final IndicesService indicesService = getInstanceFromNode(IndicesService.class);
+        final IndexService indexService = indicesService.indexServiceSafe(resolveIndex("index"));
+        final IndexShard indexShard = indexService.getShard(0);
+
+        SearchRequest searchRequest = new SearchRequest().allowPartialSearchResults(true);
+        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
+        searchRequest.source(searchSourceBuilder);
+
+        for (int i = 0; i < 5; i++) {
+            searchSourceBuilder.derivedField(
+                "field" + i,
+                "date",
+                new Script(ScriptType.INLINE, MockScriptEngine.NAME, CustomScriptPlugin.DUMMY_SCRIPT, Collections.emptyMap())
+            );
+        }
+        final ShardSearchRequest request = new ShardSearchRequest(
+            OriginalIndices.NONE,
+            searchRequest,
+            indexShard.shardId(),
+            1,
+            new AliasFilter(null, Strings.EMPTY_ARRAY),
+            1.0f,
+            -1,
+            null,
+            null
+        );
+
+        try (ReaderContext reader = createReaderContext(indexService, indexShard)) {
+            try (SearchContext context = service.createContext(reader, request, null, randomBoolean())) {
+                assertNotNull(context);
+                for (int i = 0; i < 5; i++) {
+                    DerivedFieldType derivedFieldType = (DerivedFieldType) context.getQueryShardContext().getDerivedFieldType("field" + i);
+                    assertEquals("field" + i, derivedFieldType.name());
+                    assertEquals("date", derivedFieldType.getType());
+                }
+                assertNull(context.getQueryShardContext().getDerivedFieldType("field" + 5));
+            }
+        }
+    }
+
     /**
      * test that getting more than the allowed number of script_fields throws an exception
      */
diff --git a/server/src/test/java/org/opensearch/search/aggregations/AggregationProcessorTests.java b/server/src/test/java/org/opensearch/search/aggregations/AggregationProcessorTests.java
index d68b0911d3d01..c10233a72921f 100644
--- a/server/src/test/java/org/opensearch/search/aggregations/AggregationProcessorTests.java
+++ b/server/src/test/java/org/opensearch/search/aggregations/AggregationProcessorTests.java
@@ -187,10 +187,16 @@ protected LeafSlice[] slices(List<LeafReaderContext> leaves) {
         AggregationCollectorManager collectorManager;
         if (expectedNonGlobalAggsPerSlice > 0) {
             collectorManager = (AggregationCollectorManager) context.queryCollectorManagers().get(NonGlobalAggCollectorManager.class);
+            for (Collector c : nonGlobalCollectors) {
+                context.bucketCollectorProcessor().processPostCollection(c);
+            }
             collectorManager.reduce(nonGlobalCollectors).reduce(context.queryResult());
         }
         if (expectedGlobalAggs > 0) {
             collectorManager = (AggregationCollectorManager) context.queryCollectorManagers().get(GlobalAggCollectorManager.class);
+            for (Collector c : globalCollectors) {
+                context.bucketCollectorProcessor().processPostCollection(c);
+            }
             ReduceableSearchResult result = collectorManager.reduce(globalCollectors);
             doReturn(result).when(testSearcher)
                 .search(nullable(Query.class), ArgumentMatchers.<CollectorManager<?, ReduceableSearchResult>>any());
diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java
index 2a4fbca7a8541..cf95999ec5086 100644
--- a/server/src/test/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java
+++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/histogram/DateHistogramAggregatorTests.java
@@ -38,29 +38,42 @@
 import org.apache.lucene.document.SortedNumericDocValuesField;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.search.MatchNoDocsQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.tests.index.RandomIndexWriter;
+import org.apache.lucene.tests.util.TestUtil;
 import org.opensearch.common.time.DateFormatters;
+import org.opensearch.core.common.breaker.CircuitBreaker;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
 import org.opensearch.index.mapper.DateFieldMapper;
 import org.opensearch.index.mapper.DocCountFieldMapper;
+import org.opensearch.index.mapper.MappedFieldType;
 import org.opensearch.search.aggregations.AggregationBuilder;
 import org.opensearch.search.aggregations.BucketOrder;
+import org.opensearch.search.aggregations.InternalAggregation;
+import org.opensearch.search.aggregations.MultiBucketConsumerService;
 import org.opensearch.search.aggregations.bucket.terms.StringTerms;
 import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
+import org.opensearch.search.aggregations.pipeline.PipelineAggregator;
 import org.opensearch.search.aggregations.support.AggregationInspectionHelper;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.function.Consumer;
 
 import static java.util.stream.Collectors.toList;
+import static org.opensearch.test.InternalAggregationTestCase.DEFAULT_MAX_BUCKETS;
 import static org.hamcrest.Matchers.equalTo;
 
 public class DateHistogramAggregatorTests extends DateHistogramAggregatorTestCase {
@@ -1450,6 +1463,267 @@ private void testSearchCase(
         }
     }
 
+    public void testMultiRangeTraversal() throws IOException {
+        Map<String, Integer> dataset = new HashMap<>();
+        dataset.put("2017-02-01T09:02:00.000Z", randomIntBetween(100, 2000));
+        dataset.put("2017-02-01T09:59:59.999Z", randomIntBetween(100, 2000));
+        dataset.put("2017-02-01T10:00:00.001Z", randomIntBetween(100, 2000));
+        dataset.put("2017-02-01T13:06:00.000Z", randomIntBetween(100, 2000));
+        dataset.put("2017-02-01T14:04:00.000Z", randomIntBetween(100, 2000));
+        dataset.put("2017-02-01T14:05:00.000Z", randomIntBetween(100, 2000));
+        dataset.put("2017-02-01T15:59:00.000Z", randomIntBetween(100, 2000));
+        dataset.put("2017-02-01T16:06:00.000Z", randomIntBetween(100, 2000));
+        dataset.put("2017-02-01T16:48:00.000Z", randomIntBetween(100, 2000));
+        dataset.put("2017-02-01T16:59:00.000Z", randomIntBetween(100, 2000));
+
+        testFilterRewriteCase(
+            LongPoint.newRangeQuery(AGGREGABLE_DATE, asLong("2017-01-01T09:00:00.000Z"), asLong("2017-02-01T16:00:00.000Z")),
+            dataset,
+            aggregation -> aggregation.fixedInterval(new DateHistogramInterval("60m")).field(AGGREGABLE_DATE).minDocCount(1L),
+            histogram -> {
+                List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                assertEquals(5, buckets.size());
+
+                Histogram.Bucket bucket = buckets.get(0);
+                assertEquals("2017-02-01T09:00:00.000Z", bucket.getKeyAsString());
+                int expected = dataset.get("2017-02-01T09:02:00.000Z") + dataset.get("2017-02-01T09:59:59.999Z");
+                assertEquals(expected, bucket.getDocCount());
+
+                bucket = buckets.get(1);
+                assertEquals("2017-02-01T10:00:00.000Z", bucket.getKeyAsString());
+                expected = dataset.get("2017-02-01T10:00:00.001Z");
+                assertEquals(expected, bucket.getDocCount());
+
+                bucket = buckets.get(2);
+                assertEquals("2017-02-01T13:00:00.000Z", bucket.getKeyAsString());
+                expected = dataset.get("2017-02-01T13:06:00.000Z");
+                assertEquals(expected, bucket.getDocCount());
+
+                bucket = buckets.get(3);
+                assertEquals("2017-02-01T14:00:00.000Z", bucket.getKeyAsString());
+                expected = dataset.get("2017-02-01T14:04:00.000Z") + dataset.get("2017-02-01T14:05:00.000Z");
+                assertEquals(expected, bucket.getDocCount());
+
+                bucket = buckets.get(4);
+                assertEquals("2017-02-01T15:00:00.000Z", bucket.getKeyAsString());
+                expected = dataset.get("2017-02-01T15:59:00.000Z");
+                assertEquals(expected, bucket.getDocCount());
+            },
+            false,
+            collectorCount -> assertEquals(0, (int) collectorCount),
+            true
+        );
+    }
+
+    public void testMultiRangeTraversalFixedData() throws IOException {
+        Map<String, Integer> dataset = new HashMap<>();
+        dataset.put("2017-02-01T09:02:00.000Z", 512);
+        dataset.put("2017-02-01T09:59:59.999Z", 256);
+        dataset.put("2017-02-01T10:00:00.001Z", 256);
+        dataset.put("2017-02-01T13:06:00.000Z", 512);
+        dataset.put("2017-02-01T14:04:00.000Z", 256);
+        dataset.put("2017-02-01T14:05:00.000Z", 256);
+        dataset.put("2017-02-01T15:59:00.000Z", 768);
+
+        testFilterRewriteCase(
+            LongPoint.newRangeQuery(AGGREGABLE_DATE, asLong("2017-01-01T09:00:00.000Z"), asLong("2017-02-01T14:04:01.000Z")),
+            dataset,
+            aggregation -> aggregation.fixedInterval(new DateHistogramInterval("60m")).field(AGGREGABLE_DATE).minDocCount(1L),
+            histogram -> {
+                List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                assertEquals(4, buckets.size());
+
+                Histogram.Bucket bucket = buckets.get(0);
+                assertEquals("2017-02-01T09:00:00.000Z", bucket.getKeyAsString());
+                int expected = dataset.get("2017-02-01T09:02:00.000Z") + dataset.get("2017-02-01T09:59:59.999Z");
+                assertEquals(expected, bucket.getDocCount());
+
+                bucket = buckets.get(1);
+                assertEquals("2017-02-01T10:00:00.000Z", bucket.getKeyAsString());
+                expected = dataset.get("2017-02-01T10:00:00.001Z");
+                assertEquals(expected, bucket.getDocCount());
+
+                bucket = buckets.get(2);
+                assertEquals("2017-02-01T13:00:00.000Z", bucket.getKeyAsString());
+                expected = dataset.get("2017-02-01T13:06:00.000Z");
+                assertEquals(expected, bucket.getDocCount());
+
+                bucket = buckets.get(3);
+                assertEquals("2017-02-01T14:00:00.000Z", bucket.getKeyAsString());
+                expected = dataset.get("2017-02-01T14:04:00.000Z");
+                assertEquals(expected, bucket.getDocCount());
+            },
+            false,
+            collectorCount -> assertEquals(0, (int) collectorCount),
+            false
+        );
+    }
+
+    public void testMultiRangeTraversalNotApplicable() throws IOException {
+        Map<String, Integer> dataset = new HashMap<>();
+        dataset.put("2017-02-01T09:02:00.000Z", randomIntBetween(100, 2000));
+        dataset.put("2017-02-01T09:59:59.999Z", randomIntBetween(100, 2000));
+        dataset.put("2017-02-01T10:00:00.001Z", randomIntBetween(100, 2000));
+        dataset.put("2017-02-01T13:06:00.000Z", randomIntBetween(100, 2000));
+        dataset.put("2017-02-01T14:04:00.000Z", randomIntBetween(100, 2000));
+        dataset.put("2017-02-01T14:05:00.000Z", randomIntBetween(100, 2000));
+        dataset.put("2017-02-01T15:59:00.000Z", randomIntBetween(100, 2000));
+        dataset.put("2017-02-01T16:06:00.000Z", randomIntBetween(100, 2000));
+        dataset.put("2017-02-01T16:48:00.000Z", randomIntBetween(100, 2000));
+        dataset.put("2017-02-01T16:59:00.000Z", randomIntBetween(100, 2000));
+
+        testFilterRewriteCase(
+            new MatchAllDocsQuery(),
+            dataset,
+            aggregation -> aggregation.fixedInterval(new DateHistogramInterval("60m")).field(AGGREGABLE_DATE).minDocCount(1L),
+            histogram -> {
+                List<? extends Histogram.Bucket> buckets = histogram.getBuckets();
+                assertEquals(6, buckets.size());
+
+                Histogram.Bucket bucket = buckets.get(0);
+                assertEquals("2017-02-01T09:00:00.000Z", bucket.getKeyAsString());
+                int expected = dataset.get("2017-02-01T09:02:00.000Z") + dataset.get("2017-02-01T09:59:59.999Z") + 4;
+                assertEquals(expected, bucket.getDocCount());
+
+                bucket = buckets.get(1);
+                assertEquals("2017-02-01T10:00:00.000Z", bucket.getKeyAsString());
+                expected = dataset.get("2017-02-01T10:00:00.001Z");
+                assertEquals(expected, bucket.getDocCount());
+
+                bucket = buckets.get(2);
+                assertEquals("2017-02-01T13:00:00.000Z", bucket.getKeyAsString());
+                expected = dataset.get("2017-02-01T13:06:00.000Z");
+                assertEquals(expected, bucket.getDocCount());
+
+                bucket = buckets.get(3);
+                assertEquals("2017-02-01T14:00:00.000Z", bucket.getKeyAsString());
+                expected = dataset.get("2017-02-01T14:04:00.000Z") + dataset.get("2017-02-01T14:05:00.000Z");
+                assertEquals(expected, bucket.getDocCount());
+
+                bucket = buckets.get(4);
+                assertEquals("2017-02-01T15:00:00.000Z", bucket.getKeyAsString());
+                expected = dataset.get("2017-02-01T15:59:00.000Z");
+                assertEquals(expected, bucket.getDocCount());
+
+                bucket = buckets.get(5);
+                assertEquals("2017-02-01T16:00:00.000Z", bucket.getKeyAsString());
+                expected = dataset.get("2017-02-01T16:06:00.000Z") + dataset.get("2017-02-01T16:48:00.000Z") + dataset.get(
+                    "2017-02-01T16:59:00.000Z"
+                );
+                assertEquals(expected, bucket.getDocCount());
+            },
+            true,
+            collectCount -> assertTrue(collectCount > 0),
+            true
+        );
+    }
+
+    private void testFilterRewriteCase(
+        Query query,
+        Map<String, Integer> dataset,
+        Consumer<DateHistogramAggregationBuilder> configure,
+        Consumer<InternalDateHistogram> verify,
+        boolean useDocCountField,
+        Consumer<Integer> verifyCollectCount,
+        boolean randomWrite
+    ) throws IOException {
+        DateFieldMapper.DateFieldType fieldType = aggregableDateFieldType(false, true);
+
+        try (Directory directory = newDirectory()) {
+            if (randomWrite) {
+                try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
+                    Document document = new Document();
+                    if (useDocCountField) {
+                        // add the doc count field to the first document
+                        document.add(new NumericDocValuesField(DocCountFieldMapper.NAME, 5));
+                    }
+                    for (Map.Entry<String, Integer> date : dataset.entrySet()) {
+                        for (int i = 0; i < date.getValue(); i++) {
+                            long instant = asLong(date.getKey(), fieldType);
+                            document.add(new SortedNumericDocValuesField(AGGREGABLE_DATE, instant));
+                            document.add(new LongPoint(AGGREGABLE_DATE, instant));
+                            indexWriter.addDocument(document);
+                            document.clear();
+                        }
+                    }
+                }
+            } else {
+                // use default codec so max points in leaf is fixed to 512, to cover the node level visit and compare logic
+                try (IndexWriter indexWriter = new IndexWriter(directory, new IndexWriterConfig().setCodec(TestUtil.getDefaultCodec()))) {
+                    List<Document> documents = new ArrayList<>();
+                    for (Map.Entry<String, Integer> date : dataset.entrySet()) {
+                        for (int i = 0; i < date.getValue(); i++) {
+                            Document document = new Document();
+                            if (useDocCountField) {
+                                // add the doc count field once
+                                document.add(new NumericDocValuesField(DocCountFieldMapper.NAME, 5));
+                                useDocCountField = false;
+                            }
+                            long instant = asLong(date.getKey(), fieldType);
+                            document.add(new SortedNumericDocValuesField(AGGREGABLE_DATE, instant));
+                            document.add(new LongPoint(AGGREGABLE_DATE, instant));
+                            documents.add(document);
+                        }
+                    }
+                    indexWriter.addDocuments(documents);
+                }
+            }
+
+            try (IndexReader indexReader = DirectoryReader.open(directory)) {
+                IndexSearcher indexSearcher = newSearcher(indexReader, true, true);
+
+                DateHistogramAggregationBuilder aggregationBuilder = new DateHistogramAggregationBuilder("_name");
+                if (configure != null) {
+                    configure.accept(aggregationBuilder);
+                }
+
+                CountingAggregator aggregator = createCountingAggregator(query, aggregationBuilder, indexSearcher, fieldType);
+                aggregator.preCollection();
+                indexSearcher.search(query, aggregator);
+                aggregator.postCollection();
+
+                MultiBucketConsumerService.MultiBucketConsumer reduceBucketConsumer = new MultiBucketConsumerService.MultiBucketConsumer(
+                    Integer.MAX_VALUE,
+                    new NoneCircuitBreakerService().getBreaker(CircuitBreaker.REQUEST)
+                );
+                InternalAggregation.ReduceContext context = InternalAggregation.ReduceContext.forFinalReduction(
+                    aggregator.context().bigArrays(),
+                    getMockScriptService(),
+                    reduceBucketConsumer,
+                    PipelineAggregator.PipelineTree.EMPTY
+                );
+                InternalDateHistogram topLevel = (InternalDateHistogram) aggregator.buildTopLevel();
+                InternalDateHistogram histogram = (InternalDateHistogram) topLevel.reduce(Collections.singletonList(topLevel), context);
+                doAssertReducedMultiBucketConsumer(histogram, reduceBucketConsumer);
+
+                verify.accept(histogram);
+
+                verifyCollectCount.accept(aggregator.getCollectCount().get());
+            }
+        }
+    }
+
+    protected CountingAggregator createCountingAggregator(
+        Query query,
+        AggregationBuilder builder,
+        IndexSearcher searcher,
+        MappedFieldType... fieldTypes
+    ) throws IOException {
+        return new CountingAggregator(
+            new AtomicInteger(),
+            createAggregator(
+                query,
+                builder,
+                searcher,
+                new MultiBucketConsumerService.MultiBucketConsumer(
+                    DEFAULT_MAX_BUCKETS,
+                    new NoneCircuitBreakerService().getBreaker(CircuitBreaker.REQUEST)
+                ),
+                fieldTypes
+            )
+        );
+    }
+
     private static long asLong(String dateTime) {
         return DateFormatters.from(DateFieldMapper.getDefaultDateTimeFormatter().parse(dateTime)).toInstant().toEpochMilli();
     }
diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/SignificantTextAggregatorTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/SignificantTextAggregatorTests.java
index e9b2d40fd4ede..644cee57bd5a4 100644
--- a/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/SignificantTextAggregatorTests.java
+++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/terms/SignificantTextAggregatorTests.java
@@ -50,6 +50,7 @@
 import org.opensearch.index.analysis.AnalyzerScope;
 import org.opensearch.index.analysis.NamedAnalyzer;
 import org.opensearch.index.mapper.BinaryFieldMapper;
+import org.opensearch.index.mapper.ConstantKeywordFieldMapper;
 import org.opensearch.index.mapper.FlatObjectFieldMapper;
 import org.opensearch.index.mapper.GeoPointFieldMapper;
 import org.opensearch.index.mapper.MappedFieldType;
@@ -104,7 +105,8 @@ protected List<String> unsupportedMappedFieldTypes() {
         return Arrays.asList(
             BinaryFieldMapper.CONTENT_TYPE, // binary fields are not supported because they do not have analyzers
             GeoPointFieldMapper.CONTENT_TYPE, // geopoint fields cannot use term queries
-            FlatObjectFieldMapper.CONTENT_TYPE // flat_object fields are not supported aggregations
+            FlatObjectFieldMapper.CONTENT_TYPE, // flat_object fields are not supported aggregations
+            ConstantKeywordFieldMapper.CONTENT_TYPE // binary fields are not supported because they do not have analyzers
         );
     }
 
diff --git a/server/src/test/java/org/opensearch/search/builder/SearchSourceBuilderTests.java b/server/src/test/java/org/opensearch/search/builder/SearchSourceBuilderTests.java
index 3d0e5c3eaf1c0..5b1035e24185d 100644
--- a/server/src/test/java/org/opensearch/search/builder/SearchSourceBuilderTests.java
+++ b/server/src/test/java/org/opensearch/search/builder/SearchSourceBuilderTests.java
@@ -53,6 +53,7 @@
 import org.opensearch.index.query.QueryRewriteContext;
 import org.opensearch.index.query.RandomQueryBuilder;
 import org.opensearch.index.query.Rewriteable;
+import org.opensearch.script.Script;
 import org.opensearch.search.AbstractSearchTestCase;
 import org.opensearch.search.rescore.QueryRescorerBuilder;
 import org.opensearch.search.sort.FieldSortBuilder;
@@ -311,6 +312,51 @@ public void testParseSort() throws IOException {
         }
     }
 
+    public void testDerivedFieldsParsingAndSerialization() throws IOException {
+        {
+            String restContent = "{\n"
+                + "  \"derived\": {\n"
+                + "    \"duration\": {\n"
+                + "      \"type\": \"long\",\n"
+                + "      \"script\": \"emit(doc['test'])\"\n"
+                + "    },\n"
+                + "    \"ip_from_message\": {\n"
+                + "      \"type\": \"keyword\",\n"
+                + "      \"script\": \"emit(doc['message'])\"\n"
+                + "    }\n"
+                + "  },\n"
+                + "    \"query\" : {\n"
+                + "        \"match\": { \"content\": { \"query\": \"foo bar\" }}\n"
+                + "     }\n"
+                + "}";
+
+            String expectedContent =
+                "{\"query\":{\"match\":{\"content\":{\"query\":\"foo bar\",\"operator\":\"OR\",\"prefix_length\":0,\"max_expansions\":50,\"fuzzy_transpositions\":true,\"lenient\":false,\"zero_terms_query\":\"NONE\",\"auto_generate_synonyms_phrase_query\":true,\"boost\":1.0}}},"
+                    + "\"derived\":{"
+                    + "\"duration\":{\"type\":\"long\",\"script\":\"emit(doc['test'])\"},\"ip_from_message\":{\"type\":\"keyword\",\"script\":\"emit(doc['message'])\"},\"derived_field\":{\"type\":\"keyword\",\"script\":{\"source\":\"emit(doc['message']\",\"lang\":\"painless\"}}}}";
+
+            try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) {
+                SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(parser);
+                searchSourceBuilder.derivedField("derived_field", "keyword", new Script("emit(doc['message']"));
+                searchSourceBuilder = rewrite(searchSourceBuilder);
+                assertEquals(2, searchSourceBuilder.getDerivedFieldsObject().size());
+                assertEquals(1, searchSourceBuilder.getDerivedFields().size());
+
+                try (BytesStreamOutput output = new BytesStreamOutput()) {
+                    searchSourceBuilder.writeTo(output);
+                    try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWriteableRegistry)) {
+                        SearchSourceBuilder deserializedBuilder = new SearchSourceBuilder(in);
+                        String actualContent = deserializedBuilder.toString();
+                        assertEquals(expectedContent, actualContent);
+                        assertEquals(searchSourceBuilder.hashCode(), deserializedBuilder.hashCode());
+                        assertNotSame(searchSourceBuilder, deserializedBuilder);
+                    }
+                }
+            }
+        }
+
+    }
+
     public void testAggsParsing() throws IOException {
         {
             String restContent = "{\n"
@@ -525,7 +571,7 @@ public void testParseIndicesBoost() throws IOException {
 
     public void testNegativeFromErrors() {
         IllegalArgumentException expected = expectThrows(IllegalArgumentException.class, () -> new SearchSourceBuilder().from(-2));
-        assertEquals("[from] parameter cannot be negative", expected.getMessage());
+        assertEquals("[from] parameter cannot be negative, found [-2]", expected.getMessage());
     }
 
     public void testNegativeSizeErrors() {
@@ -536,6 +582,42 @@ public void testNegativeSizeErrors() {
         assertEquals("[size] parameter cannot be negative, found [-1]", expected.getMessage());
     }
 
+    public void testParseFromAndSize() throws IOException {
+        int negativeFrom = randomIntBetween(-100, -1);
+        String restContent = " { \"from\": \"" + negativeFrom + "\"}";
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) {
+            IllegalArgumentException expected = expectThrows(
+                IllegalArgumentException.class,
+                () -> SearchSourceBuilder.fromXContent(parser)
+            );
+            assertEquals("[from] parameter cannot be negative, found [" + negativeFrom + "]", expected.getMessage());
+        }
+
+        int validFrom = randomIntBetween(0, 100);
+        restContent = " { \"from\": \"" + validFrom + "\"}";
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) {
+            SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(parser);
+            assertEquals(validFrom, searchSourceBuilder.from());
+        }
+
+        int negativeSize = randomIntBetween(-100, -1);
+        restContent = " { \"size\": \"" + negativeSize + "\"}";
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) {
+            IllegalArgumentException expected = expectThrows(
+                IllegalArgumentException.class,
+                () -> SearchSourceBuilder.fromXContent(parser)
+            );
+            assertEquals("[size] parameter cannot be negative, found [" + negativeSize + "]", expected.getMessage());
+        }
+
+        int validSize = randomIntBetween(0, 100);
+        restContent = " { \"size\": \"" + validSize + "\"}";
+        try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) {
+            SearchSourceBuilder searchSourceBuilder = SearchSourceBuilder.fromXContent(parser);
+            assertEquals(validSize, searchSourceBuilder.size());
+        }
+    }
+
     private void assertIndicesBoostParseErrorMessage(String restContent, String expectedErrorMessage) throws IOException {
         try (XContentParser parser = createParser(JsonXContent.jsonXContent, restContent)) {
             ParsingException e = expectThrows(ParsingException.class, () -> SearchSourceBuilder.fromXContent(parser));
diff --git a/server/src/test/java/org/opensearch/search/fetch/subphase/InnerHitsPhaseTests.java b/server/src/test/java/org/opensearch/search/fetch/subphase/InnerHitsPhaseTests.java
deleted file mode 100644
index 7ca5977a1c276..0000000000000
--- a/server/src/test/java/org/opensearch/search/fetch/subphase/InnerHitsPhaseTests.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.search.fetch.subphase;
-
-import org.opensearch.index.query.QueryShardContext;
-import org.opensearch.search.fetch.FetchContext;
-import org.opensearch.search.internal.SearchContext;
-import org.opensearch.search.lookup.SearchLookup;
-import org.opensearch.test.OpenSearchTestCase;
-
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-public class InnerHitsPhaseTests extends OpenSearchTestCase {
-
-    /*
-    Returns mock search context reused across test methods
-     */
-    private SearchContext getMockSearchContext(final boolean hasInnerHits) {
-        final QueryShardContext queryShardContext = mock(QueryShardContext.class);
-        when(queryShardContext.newFetchLookup()).thenReturn(mock(SearchLookup.class));
-
-        final SearchContext searchContext = mock(SearchContext.class);
-        when(searchContext.hasInnerHits()).thenReturn(hasInnerHits);
-        when(searchContext.getQueryShardContext()).thenReturn(queryShardContext);
-
-        return searchContext;
-    }
-
-    /*
-    Validates that InnerHitsPhase processor is not initialized when no inner hits
-     */
-    public void testInnerHitsNull() {
-        assertNull(new InnerHitsPhase(null).getProcessor(new FetchContext(getMockSearchContext(false))));
-    }
-
-    /*
-    Validates that InnerHitsPhase processor is initialized when inner hits are present
-     */
-    public void testInnerHitsNonNull() {
-        final SearchContext searchContext = getMockSearchContext(true);
-        when(searchContext.innerHits()).thenReturn(new InnerHitsContext());
-
-        assertNotNull(new InnerHitsPhase(null).getProcessor(new FetchContext(searchContext)));
-    }
-
-}
diff --git a/server/src/test/java/org/opensearch/search/fetch/subphase/ScriptFieldsPhaseTests.java b/server/src/test/java/org/opensearch/search/fetch/subphase/ScriptFieldsPhaseTests.java
deleted file mode 100644
index eb6338997ab9f..0000000000000
--- a/server/src/test/java/org/opensearch/search/fetch/subphase/ScriptFieldsPhaseTests.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * SPDX-License-Identifier: Apache-2.0
- *
- * The OpenSearch Contributors require contributions made to
- * this file be licensed under the Apache-2.0 license or a
- * compatible open source license.
- */
-
-package org.opensearch.search.fetch.subphase;
-
-import org.opensearch.index.query.QueryShardContext;
-import org.opensearch.search.fetch.FetchContext;
-import org.opensearch.search.internal.SearchContext;
-import org.opensearch.search.lookup.SearchLookup;
-import org.opensearch.test.OpenSearchTestCase;
-
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-public class ScriptFieldsPhaseTests extends OpenSearchTestCase {
-
-    /*
-    Returns mock search context reused across test methods
-     */
-    private SearchContext getMockSearchContext(final boolean hasScriptFields) {
-        final QueryShardContext queryShardContext = mock(QueryShardContext.class);
-        when(queryShardContext.newFetchLookup()).thenReturn(mock(SearchLookup.class));
-
-        final SearchContext searchContext = mock(SearchContext.class);
-        when(searchContext.hasScriptFields()).thenReturn(hasScriptFields);
-        when(searchContext.getQueryShardContext()).thenReturn(queryShardContext);
-
-        return searchContext;
-    }
-
-    /*
-    Validates that ScriptFieldsPhase processor is not initialized when no script fields
-     */
-    public void testScriptFieldsNull() {
-        assertNull(new ScriptFieldsPhase().getProcessor(new FetchContext(getMockSearchContext(false))));
-    }
-
-    /*
-    Validates that ScriptFieldsPhase processor is initialized when script fields are present
-     */
-    public void testScriptFieldsNonNull() {
-        final SearchContext searchContext = getMockSearchContext(true);
-        when(searchContext.scriptFields()).thenReturn(new ScriptFieldsContext());
-
-        assertNotNull(new ScriptFieldsPhase().getProcessor(new FetchContext(searchContext)));
-    }
-
-}
diff --git a/server/src/test/java/org/opensearch/search/fetch/subphase/highlight/DerivedFieldFetchAndHighlightTests.java b/server/src/test/java/org/opensearch/search/fetch/subphase/highlight/DerivedFieldFetchAndHighlightTests.java
new file mode 100644
index 0000000000000..92127da9654aa
--- /dev/null
+++ b/server/src/test/java/org/opensearch/search/fetch/subphase/highlight/DerivedFieldFetchAndHighlightTests.java
@@ -0,0 +1,366 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.search.fetch.subphase.highlight;
+
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.tests.index.RandomIndexWriter;
+import org.opensearch.Version;
+import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.common.document.DocumentField;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.xcontent.XContentFactory;
+import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.core.xcontent.MediaTypeRegistry;
+import org.opensearch.core.xcontent.XContentBuilder;
+import org.opensearch.index.IndexService;
+import org.opensearch.index.IndexSettings;
+import org.opensearch.index.mapper.ContentPath;
+import org.opensearch.index.mapper.DerivedField;
+import org.opensearch.index.mapper.DerivedFieldSupportedTypes;
+import org.opensearch.index.mapper.DerivedFieldType;
+import org.opensearch.index.mapper.Mapper;
+import org.opensearch.index.mapper.MapperService;
+import org.opensearch.index.mapper.SourceToParse;
+import org.opensearch.index.query.QueryShardContext;
+import org.opensearch.index.query.Rewriteable;
+import org.opensearch.script.MockScriptEngine;
+import org.opensearch.script.Script;
+import org.opensearch.script.ScriptEngine;
+import org.opensearch.script.ScriptModule;
+import org.opensearch.script.ScriptService;
+import org.opensearch.script.ScriptType;
+import org.opensearch.search.SearchHit;
+import org.opensearch.search.fetch.FetchContext;
+import org.opensearch.search.fetch.FetchSubPhase;
+import org.opensearch.search.fetch.FetchSubPhaseProcessor;
+import org.opensearch.search.fetch.subphase.FieldAndFormat;
+import org.opensearch.search.fetch.subphase.FieldFetcher;
+import org.opensearch.search.internal.ContextIndexSearcher;
+import org.opensearch.test.OpenSearchSingleNodeTestCase;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.singletonMap;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class DerivedFieldFetchAndHighlightTests extends OpenSearchSingleNodeTestCase {
+    private static String DERIVED_FIELD_SCRIPT_1 = "derived_field_script_1";
+    private static String DERIVED_FIELD_SCRIPT_2 = "derived_field_script_2";
+
+    private static String DERIVED_FIELD_1 = "derived_1";
+    private static String DERIVED_FIELD_2 = "derived_2";
+
+    public void testDerivedFieldFromIndexMapping() throws IOException {
+        // Create index and mapper service
+        // Define mapping for derived fields, create 2 derived fields derived_1 and derived_2
+        XContentBuilder mapping = XContentFactory.jsonBuilder()
+            .startObject()
+            .startObject("derived")
+            .startObject(DERIVED_FIELD_1)
+            .field("type", "keyword")
+            .startObject("script")
+            .field("source", DERIVED_FIELD_SCRIPT_1)
+            .field("lang", "mockscript")
+            .endObject()
+            .endObject()
+            .startObject(DERIVED_FIELD_2)
+            .field("type", "keyword")
+            .startObject("script")
+            .field("source", DERIVED_FIELD_SCRIPT_2)
+            .field("lang", "mockscript")
+            .endObject()
+            .endObject()
+            .endObject()
+            .endObject();
+
+        // Create source document with 2 fields field1 and field2.
+        // derived_1 will act on field1 and derived_2 will act on derived_2. DERIVED_FIELD_SCRIPT_1 substitutes whitespaces with _
+        XContentBuilder source = XContentFactory.jsonBuilder()
+            .startObject()
+            .field("field1", "some_text_1")
+            .field("field2", "some_text_2")
+            .endObject();
+
+        int docId = 0;
+        IndexService indexService = createIndex("test_index", Settings.EMPTY, MapperService.SINGLE_MAPPING_NAME, mapping);
+        MapperService mapperService = indexService.mapperService();
+
+        try (
+            Directory dir = newDirectory();
+            RandomIndexWriter iw = new RandomIndexWriter(random(), dir, new IndexWriterConfig(mapperService.indexAnalyzer()));
+        ) {
+            iw.addDocument(
+                mapperService.documentMapper()
+                    .parse(new SourceToParse("test_index", "0", BytesReference.bytes(source), MediaTypeRegistry.JSON))
+                    .rootDoc()
+            );
+            try (IndexReader reader = iw.getReader()) {
+                IndexSearcher searcher = newSearcher(reader);
+                LeafReaderContext context = searcher.getIndexReader().leaves().get(0);
+                QueryShardContext mockShardContext = createQueryShardContext(mapperService, searcher);
+                mockShardContext.lookup().source().setSegmentAndDocument(context, docId);
+                // mockShardContext.setDerivedFieldTypes(Map.of("derived_2", createDerivedFieldType("derived_1", "keyword"), "derived_1",
+
+                // Assert the fetch phase works for both of the derived fields
+                Map<String, DocumentField> fields = fetchFields(mockShardContext, context, "*");
+
+                // Validate FetchPhase
+                {
+                    assertEquals(fields.size(), 2);
+                    assertEquals(1, fields.get(DERIVED_FIELD_1).getValues().size());
+                    assertEquals(1, fields.get(DERIVED_FIELD_2).getValues().size());
+                    assertEquals("some_text_1", fields.get(DERIVED_FIELD_1).getValue());
+                    assertEquals("some_text_2", fields.get(DERIVED_FIELD_2).getValue());
+                }
+
+                // Create a HighlightBuilder of type unified, set its fields as derived_1 and derived_2
+                HighlightBuilder highlightBuilder = new HighlightBuilder();
+                highlightBuilder.highlighterType("unified");
+                highlightBuilder.field(DERIVED_FIELD_1);
+                highlightBuilder.field(DERIVED_FIELD_2);
+                highlightBuilder = Rewriteable.rewrite(highlightBuilder, mockShardContext);
+                SearchHighlightContext searchHighlightContext = highlightBuilder.build(mockShardContext);
+
+                // Create a HighlightPhase with highlighter defined above
+                HighlightPhase highlightPhase = new HighlightPhase(Collections.singletonMap("unified", new UnifiedHighlighter()));
+
+                // create a fetch context to be used by HighlightPhase processor
+                FetchContext fetchContext = mock(FetchContext.class);
+                when(fetchContext.mapperService()).thenReturn(mapperService);
+                when(fetchContext.getQueryShardContext()).thenReturn(mockShardContext);
+                when(fetchContext.getIndexSettings()).thenReturn(indexService.getIndexSettings());
+                when(fetchContext.searcher()).thenReturn(
+                    new ContextIndexSearcher(
+                        reader,
+                        IndexSearcher.getDefaultSimilarity(),
+                        IndexSearcher.getDefaultQueryCache(),
+                        IndexSearcher.getDefaultQueryCachingPolicy(),
+                        true,
+                        null,
+                        null
+                    )
+                );
+
+                // The query used by FetchSubPhaseProcessor to highlight is a term query on DERIVED_FIELD_1
+                FetchSubPhaseProcessor subPhaseProcessor = highlightPhase.getProcessor(
+                    fetchContext,
+                    searchHighlightContext,
+                    new TermQuery(new Term(DERIVED_FIELD_1, "some_text_1"))
+                );
+
+                // Create a search hit using the derived fields fetched above in fetch phase
+                SearchHit searchHit = new SearchHit(docId, "0", null, fields, null);
+
+                // Create a HitContext of search hit
+                FetchSubPhase.HitContext hitContext = new FetchSubPhase.HitContext(
+                    searchHit,
+                    context,
+                    docId,
+                    mockShardContext.lookup().source()
+                );
+                hitContext.sourceLookup().loadSourceIfNeeded();
+                // process the HitContext using the highlightPhase subPhaseProcessor
+                subPhaseProcessor.process(hitContext);
+
+                // Validate that 1 highlight field is present
+                assertEquals(hitContext.hit().getHighlightFields().size(), 1);
+            }
+        }
+    }
+
+    public void testDerivedFieldFromSearchMapping() throws IOException {
+        // Create source document with 2 fields field1 and field2.
+        // derived_1 will act on field1 and derived_2 will act on derived_2. DERIVED_FIELD_SCRIPT_1 substitutes whitespaces with _
+        XContentBuilder source = XContentFactory.jsonBuilder()
+            .startObject()
+            .field("field1", "some_text_1")
+            .field("field2", "some_text_2")
+            .endObject();
+
+        int docId = 0;
+
+        // Create index and mapper service
+        // We are not defining derived fields in index mapping here
+        XContentBuilder mapping = XContentFactory.jsonBuilder().startObject().endObject();
+        IndexService indexService = createIndex("test_index", Settings.EMPTY, MapperService.SINGLE_MAPPING_NAME, mapping);
+        MapperService mapperService = indexService.mapperService();
+
+        try (
+            Directory dir = newDirectory();
+            RandomIndexWriter iw = new RandomIndexWriter(random(), dir, new IndexWriterConfig(mapperService.indexAnalyzer()));
+        ) {
+            iw.addDocument(
+                mapperService.documentMapper()
+                    .parse(new SourceToParse("test_index", "0", BytesReference.bytes(source), MediaTypeRegistry.JSON))
+                    .rootDoc()
+            );
+            try (IndexReader reader = iw.getReader()) {
+                IndexSearcher searcher = newSearcher(reader);
+                LeafReaderContext context = searcher.getIndexReader().leaves().get(0);
+                QueryShardContext mockShardContext = createQueryShardContext(mapperService, searcher);
+                mockShardContext.lookup().source().setSegmentAndDocument(context, docId);
+
+                // This mock behavior is similar to adding derived fields in search request
+                mockShardContext.setDerivedFieldTypes(
+                    Map.of(
+                        DERIVED_FIELD_1,
+                        createDerivedFieldType(DERIVED_FIELD_1, "keyword", DERIVED_FIELD_SCRIPT_1),
+                        DERIVED_FIELD_2,
+                        createDerivedFieldType(DERIVED_FIELD_2, "keyword", DERIVED_FIELD_SCRIPT_2)
+                    )
+                );
+
+                // Assert the fetch phase works for both of the derived fields
+                Map<String, DocumentField> fields = fetchFields(mockShardContext, context, "derived_*");
+
+                // Validate FetchPhase
+                {
+                    assertEquals(fields.size(), 2);
+                    assertEquals(1, fields.get(DERIVED_FIELD_1).getValues().size());
+                    assertEquals(1, fields.get(DERIVED_FIELD_2).getValues().size());
+                    assertEquals("some_text_1", fields.get(DERIVED_FIELD_1).getValue());
+                    assertEquals("some_text_2", fields.get(DERIVED_FIELD_2).getValue());
+                }
+
+                // Create a HighlightBuilder of type unified, set its fields as derived_1 and derived_2
+                HighlightBuilder highlightBuilder = new HighlightBuilder();
+                highlightBuilder.highlighterType("unified");
+                highlightBuilder.field(DERIVED_FIELD_1);
+                highlightBuilder.field(DERIVED_FIELD_2);
+                highlightBuilder = Rewriteable.rewrite(highlightBuilder, mockShardContext);
+                SearchHighlightContext searchHighlightContext = highlightBuilder.build(mockShardContext);
+
+                // Create a HighlightPhase with highlighter defined above
+                HighlightPhase highlightPhase = new HighlightPhase(Collections.singletonMap("unified", new UnifiedHighlighter()));
+
+                // create a fetch context to be used by HighlightPhase processor
+                FetchContext fetchContext = mock(FetchContext.class);
+                when(fetchContext.mapperService()).thenReturn(mapperService);
+                when(fetchContext.getQueryShardContext()).thenReturn(mockShardContext);
+                when(fetchContext.getIndexSettings()).thenReturn(indexService.getIndexSettings());
+                when(fetchContext.searcher()).thenReturn(
+                    new ContextIndexSearcher(
+                        reader,
+                        IndexSearcher.getDefaultSimilarity(),
+                        IndexSearcher.getDefaultQueryCache(),
+                        IndexSearcher.getDefaultQueryCachingPolicy(),
+                        true,
+                        null,
+                        null
+                    )
+                );
+
+                // The query used by FetchSubPhaseProcessor to highlight is a term query on DERIVED_FIELD_1
+                FetchSubPhaseProcessor subPhaseProcessor = highlightPhase.getProcessor(
+                    fetchContext,
+                    searchHighlightContext,
+                    new TermQuery(new Term(DERIVED_FIELD_1, "some_text_1"))
+                );
+
+                // Create a search hit using the derived fields fetched above in fetch phase
+                SearchHit searchHit = new SearchHit(docId, "0", null, fields, null);
+
+                // Create a HitContext of search hit
+                FetchSubPhase.HitContext hitContext = new FetchSubPhase.HitContext(
+                    searchHit,
+                    context,
+                    docId,
+                    mockShardContext.lookup().source()
+                );
+                hitContext.sourceLookup().loadSourceIfNeeded();
+                // process the HitContext using the highlightPhase subPhaseProcessor
+                subPhaseProcessor.process(hitContext);
+
+                // Validate that 1 highlight field is present
+                assertEquals(hitContext.hit().getHighlightFields().size(), 1);
+            }
+        }
+    }
+
+    public static Map<String, DocumentField> fetchFields(
+        QueryShardContext queryShardContext,
+        LeafReaderContext context,
+        String fieldPattern
+    ) throws IOException {
+        List<FieldAndFormat> fields = List.of(new FieldAndFormat(fieldPattern, null));
+        FieldFetcher fieldFetcher = FieldFetcher.create(queryShardContext, queryShardContext.lookup(), fields);
+        fieldFetcher.setNextReader(context);
+        return fieldFetcher.fetch(queryShardContext.lookup().source(), Set.of());
+    }
+
+    private static QueryShardContext createQueryShardContext(MapperService mapperService, IndexSearcher indexSearcher) {
+        Settings settings = Settings.builder()
+            .put("index.version.created", Version.CURRENT)
+            .put("index.number_of_shards", 1)
+            .put("index.number_of_replicas", 0)
+            .put(IndexMetadata.SETTING_INDEX_UUID, "uuid")
+            .build();
+        IndexMetadata indexMetadata = new IndexMetadata.Builder("index").settings(settings).build();
+        IndexSettings indexSettings = new IndexSettings(indexMetadata, settings);
+
+        ScriptService scriptService = getScriptService();
+        return new QueryShardContext(
+            0,
+            indexSettings,
+            null,
+            null,
+            null,
+            mapperService,
+            null,
+            scriptService,
+            null,
+            null,
+            null,
+            indexSearcher,
+            null,
+            null,
+            null,
+            null,
+            null
+        );
+    }
+
+    private static ScriptService getScriptService() {
+        final MockScriptEngine engine = new MockScriptEngine(
+            MockScriptEngine.NAME,
+            Map.of(
+                DERIVED_FIELD_SCRIPT_1,
+                (script) -> ((String) ((Map<String, Object>) script.get("_source")).get("field1")).replace(" ", "_"),
+                DERIVED_FIELD_SCRIPT_2,
+                (script) -> ((String) ((Map<String, Object>) script.get("_source")).get("field2")).replace(" ", "_")
+            ),
+            Collections.emptyMap()
+        );
+        final Map<String, ScriptEngine> engines = singletonMap(engine.getType(), engine);
+        ScriptService scriptService = new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS);
+        return scriptService;
+    }
+
+    private DerivedFieldType createDerivedFieldType(String name, String type, String script) {
+        Mapper.BuilderContext context = mock(Mapper.BuilderContext.class);
+        when(context.path()).thenReturn(new ContentPath());
+        return new DerivedFieldType(
+            new DerivedField(name, type, new Script(ScriptType.INLINE, "mockscript", script, emptyMap())),
+            DerivedFieldSupportedTypes.getFieldMapperFromType(type, name, context),
+            DerivedFieldSupportedTypes.getIndexableFieldGeneratorType(type, name)
+        );
+    }
+}
diff --git a/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineServiceTests.java b/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineServiceTests.java
index f5851e669a2da..f5857922fdff2 100644
--- a/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineServiceTests.java
+++ b/server/src/test/java/org/opensearch/search/pipeline/SearchPipelineServiceTests.java
@@ -32,7 +32,9 @@
 import org.opensearch.cluster.ClusterChangedEvent;
 import org.opensearch.cluster.ClusterName;
 import org.opensearch.cluster.ClusterState;
+import org.opensearch.cluster.metadata.AliasMetadata;
 import org.opensearch.cluster.metadata.IndexMetadata;
+import org.opensearch.cluster.metadata.IndexNameExpressionResolver;
 import org.opensearch.cluster.metadata.Metadata;
 import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.cluster.service.ClusterService;
@@ -41,6 +43,7 @@
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.util.concurrent.AtomicArray;
 import org.opensearch.common.util.concurrent.OpenSearchExecutors;
+import org.opensearch.common.util.concurrent.ThreadContext;
 import org.opensearch.common.xcontent.XContentType;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.common.breaker.CircuitBreaker;
@@ -96,14 +99,31 @@ public Map<String, Processor.Factory<SearchPhaseResultsProcessor>> getSearchPhas
 
     private ThreadPool threadPool;
 
+    private IndexNameExpressionResolver indexNameExpressionResolver;
+
     @Before
     public void setup() {
         threadPool = mock(ThreadPool.class);
         ExecutorService executorService = OpenSearchExecutors.newDirectExecutorService();
+        indexNameExpressionResolver = new IndexNameExpressionResolver(new ThreadContext(Settings.EMPTY));
         when(threadPool.generic()).thenReturn(executorService);
         when(threadPool.executor(anyString())).thenReturn(executorService);
     }
 
+    private static IndexMetadata.Builder indexBuilder(String index) {
+        return indexBuilder(index, Settings.EMPTY);
+    }
+
+    private static IndexMetadata.Builder indexBuilder(String index, Settings additionalSettings) {
+        return IndexMetadata.builder(index).settings(addAdditionalSettings(additionalSettings));
+    }
+
+    private static Settings.Builder addAdditionalSettings(Settings additionalSettings) {
+        return settings(Version.CURRENT).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1)
+            .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)
+            .put(additionalSettings);
+    }
+
     public void testSearchPipelinePlugin() {
         Client client = mock(Client.class);
         SearchPipelineService searchPipelineService = new SearchPipelineService(
@@ -162,7 +182,7 @@ public void testResolveSearchPipelineDoesNotExist() {
         final SearchRequest searchRequest = new SearchRequest("_index").pipeline("bar");
         IllegalArgumentException e = expectThrows(
             IllegalArgumentException.class,
-            () -> searchPipelineService.resolvePipeline(searchRequest)
+            () -> searchPipelineService.resolvePipeline(searchRequest, indexNameExpressionResolver)
         );
         assertTrue(e.getMessage(), e.getMessage().contains(" not defined"));
     }
@@ -197,13 +217,13 @@ public void testResolveIndexDefaultPipeline() throws Exception {
         service.applyClusterState(cce);
 
         SearchRequest searchRequest = new SearchRequest("my_index").source(SearchSourceBuilder.searchSource().size(5));
-        PipelinedRequest pipelinedRequest = syncTransformRequest(service.resolvePipeline(searchRequest));
+        PipelinedRequest pipelinedRequest = syncTransformRequest(service.resolvePipeline(searchRequest, indexNameExpressionResolver));
         assertEquals("p1", pipelinedRequest.getPipeline().getId());
         assertEquals(10, pipelinedRequest.source().size());
 
         // Bypass the default pipeline
         searchRequest.pipeline("_none");
-        pipelinedRequest = service.resolvePipeline(searchRequest);
+        pipelinedRequest = service.resolvePipeline(searchRequest, indexNameExpressionResolver);
         assertEquals("_none", pipelinedRequest.getPipeline().getId());
         assertEquals(5, pipelinedRequest.source().size());
     }
@@ -591,23 +611,29 @@ public void testTransformRequest() throws Exception {
         );
         ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build();
         ClusterState previousState = clusterState;
-        clusterState = ClusterState.builder(clusterState)
-            .metadata(Metadata.builder().putCustom(SearchPipelineMetadata.TYPE, metadata))
-            .build();
+
+        Metadata.Builder mdBuilder = Metadata.builder()
+            .putCustom(SearchPipelineMetadata.TYPE, metadata)
+            .put(indexBuilder("my-index").putAlias(AliasMetadata.builder("barbaz")));
+
+        clusterState = ClusterState.builder(clusterState).metadata(mdBuilder).build();
+
         searchPipelineService.applyClusterState(new ClusterChangedEvent("", clusterState, previousState));
 
         int size = 10;
         SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(new TermQueryBuilder("foo", "bar")).size(size);
-        SearchRequest request = new SearchRequest("_index").source(sourceBuilder).pipeline("p1");
+        SearchRequest request = new SearchRequest("my-index").source(sourceBuilder).pipeline("p1");
 
-        PipelinedRequest pipelinedRequest = syncTransformRequest(searchPipelineService.resolvePipeline(request));
+        PipelinedRequest pipelinedRequest = syncTransformRequest(
+            searchPipelineService.resolvePipeline(request, indexNameExpressionResolver)
+        );
 
         assertEquals(2 * size, pipelinedRequest.source().size());
         assertEquals(size, request.source().size());
 
         // This request doesn't specify a pipeline, it doesn't get transformed.
-        request = new SearchRequest("_index").source(sourceBuilder);
-        pipelinedRequest = searchPipelineService.resolvePipeline(request);
+        request = new SearchRequest("my-index").source(sourceBuilder);
+        pipelinedRequest = searchPipelineService.resolvePipeline(request, indexNameExpressionResolver);
         assertEquals(size, pipelinedRequest.source().size());
     }
 
@@ -643,13 +669,13 @@ public void testTransformResponse() throws Exception {
 
         // First try without specifying a pipeline, which should be a no-op.
         SearchRequest searchRequest = new SearchRequest();
-        PipelinedRequest pipelinedRequest = searchPipelineService.resolvePipeline(searchRequest);
+        PipelinedRequest pipelinedRequest = searchPipelineService.resolvePipeline(searchRequest, indexNameExpressionResolver);
         SearchResponse notTransformedResponse = syncTransformResponse(pipelinedRequest, searchResponse);
         assertSame(searchResponse, notTransformedResponse);
 
         // Now apply a pipeline
         searchRequest = new SearchRequest().pipeline("p1");
-        pipelinedRequest = searchPipelineService.resolvePipeline(searchRequest);
+        pipelinedRequest = searchPipelineService.resolvePipeline(searchRequest, indexNameExpressionResolver);
         SearchResponse transformedResponse = syncTransformResponse(pipelinedRequest, searchResponse);
         assertEquals(size, transformedResponse.getHits().getHits().length);
         for (int i = 0; i < size; i++) {
@@ -736,7 +762,7 @@ public void testTransformSearchPhase() {
 
         // First try without specifying a pipeline, which should be a no-op.
         SearchRequest searchRequest = new SearchRequest();
-        PipelinedRequest pipelinedRequest = searchPipelineService.resolvePipeline(searchRequest);
+        PipelinedRequest pipelinedRequest = searchPipelineService.resolvePipeline(searchRequest, indexNameExpressionResolver);
         AtomicArray<SearchPhaseResult> notTransformedSearchPhaseResults = searchPhaseResults.getAtomicArray();
         pipelinedRequest.transformSearchPhaseResults(
             searchPhaseResults,
@@ -748,7 +774,7 @@ public void testTransformSearchPhase() {
 
         // Now set the pipeline as p1
         searchRequest = new SearchRequest().pipeline("p1");
-        pipelinedRequest = searchPipelineService.resolvePipeline(searchRequest);
+        pipelinedRequest = searchPipelineService.resolvePipeline(searchRequest, indexNameExpressionResolver);
 
         pipelinedRequest.transformSearchPhaseResults(
             searchPhaseResults,
@@ -766,7 +792,7 @@ public void testTransformSearchPhase() {
 
         // Check Processor doesn't run for between other phases
         searchRequest = new SearchRequest().pipeline("p1");
-        pipelinedRequest = searchPipelineService.resolvePipeline(searchRequest);
+        pipelinedRequest = searchPipelineService.resolvePipeline(searchRequest, indexNameExpressionResolver);
         AtomicArray<SearchPhaseResult> notTransformedSearchPhaseResult = searchPhaseResults.getAtomicArray();
         pipelinedRequest.transformSearchPhaseResults(
             searchPhaseResults,
@@ -916,7 +942,9 @@ public void testInlinePipeline() throws Exception {
         SearchRequest searchRequest = new SearchRequest().source(sourceBuilder);
 
         // Verify pipeline
-        PipelinedRequest pipelinedRequest = syncTransformRequest(searchPipelineService.resolvePipeline(searchRequest));
+        PipelinedRequest pipelinedRequest = syncTransformRequest(
+            searchPipelineService.resolvePipeline(searchRequest, indexNameExpressionResolver)
+        );
         Pipeline pipeline = pipelinedRequest.getPipeline();
         assertEquals(SearchPipelineService.AD_HOC_PIPELINE_ID, pipeline.getId());
         assertEquals(1, pipeline.getSearchRequestProcessors().size());
@@ -961,7 +989,10 @@ public void testExceptionOnPipelineCreation() {
         SearchRequest searchRequest = new SearchRequest().source(sourceBuilder);
 
         // Exception thrown when creating the pipeline
-        expectThrows(SearchPipelineProcessingException.class, () -> searchPipelineService.resolvePipeline(searchRequest));
+        expectThrows(
+            SearchPipelineProcessingException.class,
+            () -> searchPipelineService.resolvePipeline(searchRequest, indexNameExpressionResolver)
+        );
 
     }
 
@@ -989,7 +1020,7 @@ public void testExceptionOnRequestProcessing() {
         // Exception thrown when processing the request
         expectThrows(
             SearchPipelineProcessingException.class,
-            () -> syncTransformRequest(searchPipelineService.resolvePipeline(searchRequest))
+            () -> syncTransformRequest(searchPipelineService.resolvePipeline(searchRequest, indexNameExpressionResolver))
         );
     }
 
@@ -1014,7 +1045,7 @@ public void testExceptionOnResponseProcessing() throws Exception {
         SearchSourceBuilder sourceBuilder = SearchSourceBuilder.searchSource().size(100).searchPipelineSource(pipelineSourceMap);
         SearchRequest searchRequest = new SearchRequest().source(sourceBuilder);
 
-        PipelinedRequest pipelinedRequest = searchPipelineService.resolvePipeline(searchRequest);
+        PipelinedRequest pipelinedRequest = searchPipelineService.resolvePipeline(searchRequest, indexNameExpressionResolver);
 
         SearchResponse response = new SearchResponse(null, null, 0, 0, 0, 0, null, null);
         // Exception thrown when processing response
@@ -1052,7 +1083,7 @@ public void testCatchExceptionOnRequestProcessing() throws Exception {
                     "The exception from request processor [throwing_request] in the search pipeline [_ad_hoc_pipeline] was ignored"
                 )
             );
-            syncTransformRequest(searchPipelineService.resolvePipeline(searchRequest));
+            syncTransformRequest(searchPipelineService.resolvePipeline(searchRequest, indexNameExpressionResolver));
             mockAppender.assertAllExpectationsMatched();
         }
     }
@@ -1078,7 +1109,7 @@ public void testCatchExceptionOnResponseProcessing() throws Exception {
         SearchSourceBuilder sourceBuilder = SearchSourceBuilder.searchSource().size(100).searchPipelineSource(pipelineSourceMap);
         SearchRequest searchRequest = new SearchRequest().source(sourceBuilder);
 
-        PipelinedRequest pipelinedRequest = searchPipelineService.resolvePipeline(searchRequest);
+        PipelinedRequest pipelinedRequest = searchPipelineService.resolvePipeline(searchRequest, indexNameExpressionResolver);
 
         SearchResponse response = new SearchResponse(null, null, 0, 0, 0, 0, null, null);
 
@@ -1122,15 +1153,27 @@ public void testStats() throws Exception {
         SearchRequest request = new SearchRequest();
         SearchResponse response = new SearchResponse(null, null, 0, 0, 0, 0, null, null);
 
-        syncExecutePipeline(searchPipelineService.resolvePipeline(request.pipeline("good_request_pipeline")), response);
+        syncExecutePipeline(
+            searchPipelineService.resolvePipeline(request.pipeline("good_request_pipeline"), indexNameExpressionResolver),
+            response
+        );
         expectThrows(
             SearchPipelineProcessingException.class,
-            () -> syncExecutePipeline(searchPipelineService.resolvePipeline(request.pipeline("bad_request_pipeline")), response)
+            () -> syncExecutePipeline(
+                searchPipelineService.resolvePipeline(request.pipeline("bad_request_pipeline"), indexNameExpressionResolver),
+                response
+            )
+        );
+        syncExecutePipeline(
+            searchPipelineService.resolvePipeline(request.pipeline("good_response_pipeline"), indexNameExpressionResolver),
+            response
         );
-        syncExecutePipeline(searchPipelineService.resolvePipeline(request.pipeline("good_response_pipeline")), response);
         expectThrows(
             SearchPipelineProcessingException.class,
-            () -> syncExecutePipeline(searchPipelineService.resolvePipeline(request.pipeline("bad_response_pipeline")), response)
+            () -> syncExecutePipeline(
+                searchPipelineService.resolvePipeline(request.pipeline("bad_response_pipeline"), indexNameExpressionResolver),
+                response
+            )
         );
 
         SearchPipelineStats stats = searchPipelineService.stats();
@@ -1208,12 +1251,24 @@ public void testStatsEnabledIgnoreFailure() throws Exception {
         SearchRequest request = new SearchRequest();
         SearchResponse response = new SearchResponse(null, null, 0, 0, 0, 0, null, null);
 
-        syncExecutePipeline(searchPipelineService.resolvePipeline(request.pipeline("good_request_pipeline")), response);
+        syncExecutePipeline(
+            searchPipelineService.resolvePipeline(request.pipeline("good_request_pipeline"), indexNameExpressionResolver),
+            response
+        );
         // Caught Exception here
-        syncExecutePipeline(searchPipelineService.resolvePipeline(request.pipeline("bad_request_pipeline")), response);
-        syncExecutePipeline(searchPipelineService.resolvePipeline(request.pipeline("good_response_pipeline")), response);
+        syncExecutePipeline(
+            searchPipelineService.resolvePipeline(request.pipeline("bad_request_pipeline"), indexNameExpressionResolver),
+            response
+        );
+        syncExecutePipeline(
+            searchPipelineService.resolvePipeline(request.pipeline("good_response_pipeline"), indexNameExpressionResolver),
+            response
+        );
         // Caught Exception here
-        syncExecutePipeline(searchPipelineService.resolvePipeline(request.pipeline("bad_response_pipeline")), response);
+        syncExecutePipeline(
+            searchPipelineService.resolvePipeline(request.pipeline("bad_response_pipeline"), indexNameExpressionResolver),
+            response
+        );
 
         // when ignoreFailure enabled, the search pipelines will all succeed.
         SearchPipelineStats stats = searchPipelineService.stats();
@@ -1355,7 +1410,10 @@ public void testAdHocRejectingProcessor() {
 
         SearchSourceBuilder sourceBuilder = SearchSourceBuilder.searchSource().searchPipelineSource(pipelineSourceMap);
         SearchRequest searchRequest = new SearchRequest().source(sourceBuilder);
-        expectThrows(SearchPipelineProcessingException.class, () -> searchPipelineService.resolvePipeline(searchRequest));
+        expectThrows(
+            SearchPipelineProcessingException.class,
+            () -> searchPipelineService.resolvePipeline(searchRequest, indexNameExpressionResolver)
+        );
     }
 
     public void testExtraParameterInProcessorConfig() {
@@ -1369,7 +1427,7 @@ public void testExtraParameterInProcessorConfig() {
         SearchSourceBuilder sourceBuilder = SearchSourceBuilder.searchSource().searchPipelineSource(pipelineSourceMap);
         SearchRequest searchRequest = new SearchRequest().source(sourceBuilder);
         try {
-            searchPipelineService.resolvePipeline(searchRequest);
+            searchPipelineService.resolvePipeline(searchRequest, indexNameExpressionResolver);
             fail("Exception should have been thrown");
         } catch (SearchPipelineProcessingException e) {
             assertTrue(
@@ -1462,10 +1520,183 @@ public void testStatefulProcessors() throws Exception {
             .build();
         searchPipelineService.applyClusterState(new ClusterChangedEvent("", clusterState, previousState));
 
-        PipelinedRequest request = searchPipelineService.resolvePipeline(new SearchRequest().pipeline("p1"));
+        PipelinedRequest request = searchPipelineService.resolvePipeline(new SearchRequest().pipeline("p1"), indexNameExpressionResolver);
         assertNull(contextHolder.get());
         syncExecutePipeline(request, new SearchResponse(null, null, 0, 0, 0, 0, null, null));
         assertNotNull(contextHolder.get());
         assertEquals("b", contextHolder.get());
     }
+
+    public void testDefaultPipelineForMultipleIndices() throws Exception {
+        SearchPipelineService service = createWithProcessors();
+
+        SearchPipelineMetadata metadata = new SearchPipelineMetadata(
+            Map.of(
+                "p1",
+                new PipelineConfiguration(
+                    "p1",
+                    new BytesArray("{\"request_processors\" : [ { \"scale_request_size\": { \"scale\" : 2 } } ] }"),
+                    MediaTypeRegistry.JSON
+                )
+            )
+        );
+
+        Settings defaultPipelineSetting = Settings.builder()
+            .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
+            .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
+            .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), Version.CURRENT)
+            .put(IndexSettings.DEFAULT_SEARCH_PIPELINE.getKey(), "p1")
+            .build();
+
+        Metadata.Builder mdBuilder = Metadata.builder()
+            .put(indexBuilder("foo", defaultPipelineSetting).putAlias(AliasMetadata.builder("bar")))
+            .put(indexBuilder("foobar", defaultPipelineSetting).putAlias(AliasMetadata.builder("bar")))
+            .put(indexBuilder("foofoo-closed", defaultPipelineSetting).putAlias(AliasMetadata.builder("bar")))
+            .put(indexBuilder("foofoo", defaultPipelineSetting).putAlias(AliasMetadata.builder("bar")));
+
+        ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build();
+        ClusterState previousState = clusterState;
+
+        clusterState = ClusterState.builder(clusterState).metadata(mdBuilder.putCustom(SearchPipelineMetadata.TYPE, metadata)).build();
+
+        ClusterChangedEvent cce = new ClusterChangedEvent("", clusterState, previousState);
+        service.applyClusterState(cce);
+
+        SearchRequest searchRequest = new SearchRequest("bar").source(SearchSourceBuilder.searchSource().size(5));
+        PipelinedRequest pipelinedRequest = syncTransformRequest(service.resolvePipeline(searchRequest, indexNameExpressionResolver));
+        assertEquals("p1", pipelinedRequest.getPipeline().getId());
+        assertEquals(10, pipelinedRequest.source().size());
+
+        // Bypass the default pipeline
+        searchRequest.pipeline("_none");
+        pipelinedRequest = service.resolvePipeline(searchRequest, indexNameExpressionResolver);
+        assertEquals("_none", pipelinedRequest.getPipeline().getId());
+        assertEquals(5, pipelinedRequest.source().size());
+    }
+
+    public void testDifferentDefaultPipelineForMultipleIndices() throws Exception {
+        SearchPipelineService service = createWithProcessors();
+
+        SearchPipelineMetadata metadata = new SearchPipelineMetadata(
+            Map.of(
+                "p1",
+                new PipelineConfiguration(
+                    "p1",
+                    new BytesArray("{\"request_processors\" : [ { \"scale_request_size\": { \"scale\" : 2 } } ] }"),
+                    MediaTypeRegistry.JSON
+                ),
+
+                "p2",
+                new PipelineConfiguration(
+                    "p2",
+                    new BytesArray("{\"request_processors\" : [ { \"scale_request_size\": { \"scale\" : 1 } } ] }"),
+                    MediaTypeRegistry.JSON
+                )
+            )
+        );
+
+        Settings defaultPipelineSetting1 = Settings.builder()
+            .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
+            .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
+            .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), Version.CURRENT)
+            .put(IndexSettings.DEFAULT_SEARCH_PIPELINE.getKey(), "p1")
+            .build();
+
+        Settings defaultPipelineSetting2 = Settings.builder()
+            .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
+            .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
+            .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), Version.CURRENT)
+            .put(IndexSettings.DEFAULT_SEARCH_PIPELINE.getKey(), "p2")
+            .build();
+
+        Metadata.Builder mdBuilder = Metadata.builder()
+            .put(indexBuilder("foo", defaultPipelineSetting1).putAlias(AliasMetadata.builder("bar")))
+            .put(indexBuilder("foobar", defaultPipelineSetting1).putAlias(AliasMetadata.builder("bar")))
+            .put(indexBuilder("foofoo-closed", defaultPipelineSetting1).putAlias(AliasMetadata.builder("bar")))
+            .put(indexBuilder("foofoo", defaultPipelineSetting2).putAlias(AliasMetadata.builder("bar")));
+
+        ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build();
+        ClusterState previousState = clusterState;
+
+        clusterState = ClusterState.builder(clusterState).metadata(mdBuilder.putCustom(SearchPipelineMetadata.TYPE, metadata)).build();
+
+        ClusterChangedEvent cce = new ClusterChangedEvent("", clusterState, previousState);
+        service.applyClusterState(cce);
+
+        SearchRequest searchRequest = new SearchRequest("bar").source(SearchSourceBuilder.searchSource().size(5));
+        PipelinedRequest pipelinedRequest = syncTransformRequest(service.resolvePipeline(searchRequest, indexNameExpressionResolver));
+        assertEquals("_none", pipelinedRequest.getPipeline().getId());
+        assertEquals(5, pipelinedRequest.source().size());
+    }
+
+    public void testNoIndexResolveIndexDefaultPipeline() throws Exception {
+        SearchPipelineService service = createWithProcessors();
+
+        SearchPipelineMetadata metadata = new SearchPipelineMetadata(
+            Map.of(
+                "p1",
+                new PipelineConfiguration(
+                    "p1",
+                    new BytesArray("{\"request_processors\" : [ { \"scale_request_size\": { \"scale\" : 2 } } ] }"),
+                    MediaTypeRegistry.JSON
+                )
+            )
+        );
+        Settings defaultPipelineSetting = Settings.builder()
+            .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
+            .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
+            .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), Version.CURRENT)
+            .put(IndexSettings.DEFAULT_SEARCH_PIPELINE.getKey(), "p1")
+            .build();
+        IndexMetadata indexMetadata = new IndexMetadata.Builder("my_index").settings(defaultPipelineSetting).build();
+        ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build();
+        ClusterState previousState = clusterState;
+        clusterState = ClusterState.builder(clusterState)
+            .metadata(Metadata.builder().put(indexMetadata, false).putCustom(SearchPipelineMetadata.TYPE, metadata))
+            .build();
+
+        ClusterChangedEvent cce = new ClusterChangedEvent("", clusterState, previousState);
+        service.applyClusterState(cce);
+
+        SearchRequest searchRequest = new SearchRequest().source(SearchSourceBuilder.searchSource().size(5));
+        PipelinedRequest pipelinedRequest = syncTransformRequest(service.resolvePipeline(searchRequest, indexNameExpressionResolver));
+        assertEquals("_none", pipelinedRequest.getPipeline().getId());
+        assertEquals(5, pipelinedRequest.source().size());
+    }
+
+    public void testInvalidIndexResolveIndexDefaultPipeline() throws Exception {
+        SearchPipelineService service = createWithProcessors();
+
+        SearchPipelineMetadata metadata = new SearchPipelineMetadata(
+            Map.of(
+                "p1",
+                new PipelineConfiguration(
+                    "p1",
+                    new BytesArray("{\"request_processors\" : [ { \"scale_request_size\": { \"scale\" : 2 } } ] }"),
+                    MediaTypeRegistry.JSON
+                )
+            )
+        );
+        Settings defaultPipelineSetting = Settings.builder()
+            .put(IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING.getKey(), 1)
+            .put(IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.getKey(), 0)
+            .put(IndexMetadata.SETTING_INDEX_VERSION_CREATED.getKey(), Version.CURRENT)
+            .put(IndexSettings.DEFAULT_SEARCH_PIPELINE.getKey(), "p1")
+            .build();
+        IndexMetadata indexMetadata = new IndexMetadata.Builder("my_index").settings(defaultPipelineSetting).build();
+        ClusterState clusterState = ClusterState.builder(new ClusterName("_name")).build();
+        ClusterState previousState = clusterState;
+        clusterState = ClusterState.builder(clusterState)
+            .metadata(Metadata.builder().put(indexMetadata, false).putCustom(SearchPipelineMetadata.TYPE, metadata))
+            .build();
+
+        ClusterChangedEvent cce = new ClusterChangedEvent("", clusterState, previousState);
+        service.applyClusterState(cce);
+
+        SearchRequest searchRequest = new SearchRequest("xyz").source(SearchSourceBuilder.searchSource().size(5));
+        PipelinedRequest pipelinedRequest = syncTransformRequest(service.resolvePipeline(searchRequest, indexNameExpressionResolver));
+        assertEquals("_none", pipelinedRequest.getPipeline().getId());
+        assertEquals(5, pipelinedRequest.source().size());
+    }
+
 }
diff --git a/server/src/test/java/org/opensearch/search/pit/RestDeletePitActionTests.java b/server/src/test/java/org/opensearch/search/pit/RestDeletePitActionTests.java
index b60541825e3ed..448ba9e5a8cd7 100644
--- a/server/src/test/java/org/opensearch/search/pit/RestDeletePitActionTests.java
+++ b/server/src/test/java/org/opensearch/search/pit/RestDeletePitActionTests.java
@@ -82,31 +82,6 @@ public void deletePits(DeletePitRequest request, ActionListener<DeletePitRespons
         }
     }
 
-    public void testDeleteAllPitWithBody() {
-        SetOnce<Boolean> pitCalled = new SetOnce<>();
-        try (NodeClient nodeClient = new NoOpNodeClient(this.getTestName()) {
-            @Override
-            public void deletePits(DeletePitRequest request, ActionListener<DeletePitResponse> listener) {
-                pitCalled.set(true);
-                assertThat(request.getPitIds(), hasSize(1));
-                assertThat(request.getPitIds().get(0), equalTo("_all"));
-            }
-        }) {
-            RestDeletePitAction action = new RestDeletePitAction();
-            RestRequest request = new FakeRestRequest.Builder(xContentRegistry()).withContent(
-                new BytesArray("{\"pit_id\": [\"BODY\"]}"),
-                MediaTypeRegistry.JSON
-            ).withPath("/_all").build();
-            FakeRestChannel channel = new FakeRestChannel(request, false, 0);
-
-            IllegalArgumentException ex = expectThrows(
-                IllegalArgumentException.class,
-                () -> action.handleRequest(request, channel, nodeClient)
-            );
-            assertTrue(ex.getMessage().contains("request [GET /_all] does not support having a body"));
-        }
-    }
-
     public void testDeletePitQueryStringParamsShouldThrowException() {
         SetOnce<Boolean> pitCalled = new SetOnce<>();
         try (NodeClient nodeClient = new NoOpNodeClient(this.getTestName()) {
diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java
index 0614a887c13ec..3a07aa413e2a2 100644
--- a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java
+++ b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java
@@ -66,8 +66,6 @@
 import org.opensearch.action.admin.cluster.state.ClusterStateRequest;
 import org.opensearch.action.admin.cluster.state.ClusterStateResponse;
 import org.opensearch.action.admin.cluster.state.TransportClusterStateAction;
-import org.opensearch.action.admin.cluster.state.term.GetTermVersionAction;
-import org.opensearch.action.admin.cluster.state.term.TransportGetTermVersionAction;
 import org.opensearch.action.admin.indices.create.CreateIndexAction;
 import org.opensearch.action.admin.indices.create.CreateIndexRequest;
 import org.opensearch.action.admin.indices.create.CreateIndexResponse;
@@ -105,6 +103,8 @@
 import org.opensearch.action.support.PlainActionFuture;
 import org.opensearch.action.support.TransportAction;
 import org.opensearch.action.support.WriteRequest;
+import org.opensearch.action.support.clustermanager.term.GetTermVersionAction;
+import org.opensearch.action.support.clustermanager.term.TransportGetTermVersionAction;
 import org.opensearch.action.support.master.AcknowledgedResponse;
 import org.opensearch.action.update.UpdateHelper;
 import org.opensearch.client.AdminClient;
@@ -112,6 +112,7 @@
 import org.opensearch.cluster.ClusterChangedEvent;
 import org.opensearch.cluster.ClusterInfo;
 import org.opensearch.cluster.ClusterInfoService;
+import org.opensearch.cluster.ClusterManagerMetrics;
 import org.opensearch.cluster.ClusterModule;
 import org.opensearch.cluster.ClusterName;
 import org.opensearch.cluster.ClusterState;
@@ -192,6 +193,7 @@
 import org.opensearch.index.store.RemoteSegmentStoreDirectoryFactory;
 import org.opensearch.index.store.remote.filecache.FileCache;
 import org.opensearch.index.store.remote.filecache.FileCacheStats;
+import org.opensearch.indices.DefaultRemoteStoreSettings;
 import org.opensearch.indices.IndicesModule;
 import org.opensearch.indices.IndicesService;
 import org.opensearch.indices.ShardLimitValidator;
@@ -1921,7 +1923,13 @@ private final class TestClusterNode {
                     settings,
                     clusterSettings,
                     clusterManagerService,
-                    new ClusterApplierService(node.getName(), settings, clusterSettings, threadPool, NoopMetricsRegistry.INSTANCE) {
+                    new ClusterApplierService(
+                        node.getName(),
+                        settings,
+                        clusterSettings,
+                        threadPool,
+                        new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE)
+                    ) {
                         @Override
                         protected PrioritizedOpenSearchThreadPoolExecutor createThreadPoolExecutor() {
                             return new MockSinglePrioritizingExecutor(node.getName(), deterministicTaskQueue, threadPool);
@@ -2077,7 +2085,8 @@ public void onFailure(final Exception e) {
                     null,
                     new RemoteStoreStatsTrackerFactory(clusterService, settings),
                     DefaultRecoverySettings.INSTANCE,
-                    new CacheModule(new ArrayList<>(), settings).getCacheService()
+                    new CacheModule(new ArrayList<>(), settings).getCacheService(),
+                    DefaultRemoteStoreSettings.INSTANCE
                 );
                 final RecoverySettings recoverySettings = new RecoverySettings(settings, clusterSettings);
                 snapshotShardsService = new SnapshotShardsService(
@@ -2161,7 +2170,8 @@ public void onFailure(final Exception e) {
                     namedXContentRegistry,
                     systemIndices,
                     false,
-                    new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings())
+                    new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings()),
+                    DefaultRemoteStoreSettings.INSTANCE
                 );
                 actions.put(
                     CreateIndexAction.INSTANCE,
diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotUtilsTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotUtilsTests.java
index 14e711e03a345..871f2cdd88ea1 100644
--- a/server/src/test/java/org/opensearch/snapshots/SnapshotUtilsTests.java
+++ b/server/src/test/java/org/opensearch/snapshots/SnapshotUtilsTests.java
@@ -46,6 +46,7 @@
 import java.util.Map;
 
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_VERSION_CREATED;
+import static org.opensearch.common.util.IndexUtils.filterIndices;
 import static org.hamcrest.Matchers.containsInAnyOrder;
 
 public class SnapshotUtilsTests extends OpenSearchTestCase {
@@ -94,7 +95,7 @@ private void assertIndexNameFiltering(String[] indices, String[] filter, String[
 
     private void assertIndexNameFiltering(String[] indices, String[] filter, IndicesOptions indicesOptions, String[] expected) {
         List<String> indicesList = Arrays.asList(indices);
-        List<String> actual = SnapshotUtils.filterIndices(indicesList, filter, indicesOptions);
+        List<String> actual = filterIndices(indicesList, filter, indicesOptions);
         assertThat(actual, containsInAnyOrder(expected));
     }
 
diff --git a/server/src/test/java/org/opensearch/transport/InboundAggregatorTests.java b/server/src/test/java/org/opensearch/transport/InboundAggregatorTests.java
index 2dd98a8efe2a3..4ac78366360d7 100644
--- a/server/src/test/java/org/opensearch/transport/InboundAggregatorTests.java
+++ b/server/src/test/java/org/opensearch/transport/InboundAggregatorTests.java
@@ -42,6 +42,7 @@
 import org.opensearch.core.common.breaker.CircuitBreakingException;
 import org.opensearch.core.common.bytes.BytesArray;
 import org.opensearch.test.OpenSearchTestCase;
+import org.opensearch.transport.nativeprotocol.NativeInboundMessage;
 import org.junit.Before;
 
 import java.io.IOException;
@@ -107,7 +108,7 @@ public void testInboundAggregation() throws IOException {
         }
 
         // Signal EOS
-        InboundMessage aggregated = aggregator.finishAggregation();
+        NativeInboundMessage aggregated = aggregator.finishAggregation();
 
         assertThat(aggregated, notNullValue());
         assertFalse(aggregated.isPing());
@@ -138,7 +139,7 @@ public void testInboundUnknownAction() throws IOException {
         assertEquals(0, content.refCount());
 
         // Signal EOS
-        InboundMessage aggregated = aggregator.finishAggregation();
+        NativeInboundMessage aggregated = aggregator.finishAggregation();
 
         assertThat(aggregated, notNullValue());
         assertTrue(aggregated.isShortCircuit());
@@ -161,7 +162,7 @@ public void testCircuitBreak() throws IOException {
         content1.close();
 
         // Signal EOS
-        InboundMessage aggregated1 = aggregator.finishAggregation();
+        NativeInboundMessage aggregated1 = aggregator.finishAggregation();
 
         assertEquals(0, content1.refCount());
         assertThat(aggregated1, notNullValue());
@@ -180,7 +181,7 @@ public void testCircuitBreak() throws IOException {
         content2.close();
 
         // Signal EOS
-        InboundMessage aggregated2 = aggregator.finishAggregation();
+        NativeInboundMessage aggregated2 = aggregator.finishAggregation();
 
         assertEquals(1, content2.refCount());
         assertThat(aggregated2, notNullValue());
@@ -199,7 +200,7 @@ public void testCircuitBreak() throws IOException {
         content3.close();
 
         // Signal EOS
-        InboundMessage aggregated3 = aggregator.finishAggregation();
+        NativeInboundMessage aggregated3 = aggregator.finishAggregation();
 
         assertEquals(1, content3.refCount());
         assertThat(aggregated3, notNullValue());
@@ -263,7 +264,7 @@ public void testFinishAggregationWillFinishHeader() throws IOException {
             content.close();
 
             // Signal EOS
-            InboundMessage aggregated = aggregator.finishAggregation();
+            NativeInboundMessage aggregated = aggregator.finishAggregation();
 
             assertThat(aggregated, notNullValue());
             assertFalse(header.needsToReadVariableHeader());
diff --git a/server/src/test/java/org/opensearch/transport/InboundDecoderTests.java b/server/src/test/java/org/opensearch/transport/InboundDecoderTests.java
index 4d671443f396e..47ae8f9783ea3 100644
--- a/server/src/test/java/org/opensearch/transport/InboundDecoderTests.java
+++ b/server/src/test/java/org/opensearch/transport/InboundDecoderTests.java
@@ -39,19 +39,29 @@
 import org.opensearch.common.util.PageCacheRecycler;
 import org.opensearch.common.util.concurrent.ThreadContext;
 import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.core.common.io.stream.Writeable;
 import org.opensearch.core.transport.TransportMessage;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.test.VersionUtils;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collections;
 
 import static org.hamcrest.Matchers.hasItems;
 
-public class InboundDecoderTests extends OpenSearchTestCase {
+public abstract class InboundDecoderTests extends OpenSearchTestCase {
 
-    private ThreadContext threadContext;
+    protected ThreadContext threadContext;
+
+    protected abstract BytesReference serialize(
+        boolean isRequest,
+        Version version,
+        boolean handshake,
+        boolean compress,
+        String action,
+        long requestId,
+        Writeable transportMessage
+    ) throws IOException;
 
     @Override
     public void setUp() throws Exception {
@@ -65,36 +75,16 @@ public void testDecode() throws IOException {
         long requestId = randomNonNegativeLong();
         final String headerKey = randomAlphaOfLength(10);
         final String headerValue = randomAlphaOfLength(20);
+        TransportMessage transportMessage;
         if (isRequest) {
             threadContext.putHeader(headerKey, headerValue);
+            transportMessage = new TestRequest(randomAlphaOfLength(100));
         } else {
             threadContext.addResponseHeader(headerKey, headerValue);
-        }
-        OutboundMessage message;
-        if (isRequest) {
-            message = new OutboundMessage.Request(
-                threadContext,
-                new String[0],
-                new TestRequest(randomAlphaOfLength(100)),
-                Version.CURRENT,
-                action,
-                requestId,
-                false,
-                false
-            );
-        } else {
-            message = new OutboundMessage.Response(
-                threadContext,
-                Collections.emptySet(),
-                new TestResponse(randomAlphaOfLength(100)),
-                Version.CURRENT,
-                requestId,
-                false,
-                false
-            );
+            transportMessage = new TestResponse(randomAlphaOfLength(100));
         }
 
-        final BytesReference totalBytes = message.serialize(new BytesStreamOutput());
+        final BytesReference totalBytes = serialize(isRequest, Version.CURRENT, false, false, action, requestId, transportMessage);
         int totalHeaderSize = TcpHeader.headerSize(Version.CURRENT) + totalBytes.getInt(TcpHeader.VARIABLE_HEADER_SIZE_POSITION);
         final BytesReference messageBytes = totalBytes.slice(totalHeaderSize, totalBytes.length() - totalHeaderSize);
 
@@ -142,18 +132,16 @@ public void testDecodeHandshakeCompatibility() throws IOException {
         final String headerValue = randomAlphaOfLength(20);
         threadContext.putHeader(headerKey, headerValue);
         Version handshakeCompatVersion = Version.CURRENT.minimumCompatibilityVersion().minimumCompatibilityVersion();
-        OutboundMessage message = new OutboundMessage.Request(
-            threadContext,
-            new String[0],
-            new TestRequest(randomAlphaOfLength(100)),
+
+        final BytesReference bytes = serialize(
+            true,
             handshakeCompatVersion,
+            true,
+            false,
             action,
             requestId,
-            true,
-            false
+            new TestRequest(randomAlphaOfLength(100))
         );
-
-        final BytesReference bytes = message.serialize(new BytesStreamOutput());
         int totalHeaderSize = TcpHeader.headerSize(handshakeCompatVersion) + bytes.getInt(TcpHeader.VARIABLE_HEADER_SIZE_POSITION);
 
         InboundDecoder decoder = new InboundDecoder(Version.CURRENT, PageCacheRecycler.NON_RECYCLING_INSTANCE);
@@ -183,34 +171,14 @@ public void testCompressedDecode() throws IOException {
         } else {
             threadContext.addResponseHeader(headerKey, headerValue);
         }
-        OutboundMessage message;
         TransportMessage transportMessage;
         if (isRequest) {
             transportMessage = new TestRequest(randomAlphaOfLength(100));
-            message = new OutboundMessage.Request(
-                threadContext,
-                new String[0],
-                transportMessage,
-                Version.CURRENT,
-                action,
-                requestId,
-                false,
-                true
-            );
         } else {
             transportMessage = new TestResponse(randomAlphaOfLength(100));
-            message = new OutboundMessage.Response(
-                threadContext,
-                Collections.emptySet(),
-                transportMessage,
-                Version.CURRENT,
-                requestId,
-                false,
-                true
-            );
         }
 
-        final BytesReference totalBytes = message.serialize(new BytesStreamOutput());
+        final BytesReference totalBytes = serialize(isRequest, Version.CURRENT, false, true, action, requestId, transportMessage);
         final BytesStreamOutput out = new BytesStreamOutput();
         transportMessage.writeTo(out);
         final BytesReference uncompressedBytes = out.bytes();
@@ -260,18 +228,16 @@ public void testCompressedDecodeHandshakeCompatibility() throws IOException {
         final String headerValue = randomAlphaOfLength(20);
         threadContext.putHeader(headerKey, headerValue);
         Version handshakeCompatVersion = Version.CURRENT.minimumCompatibilityVersion().minimumCompatibilityVersion();
-        OutboundMessage message = new OutboundMessage.Request(
-            threadContext,
-            new String[0],
-            new TestRequest(randomAlphaOfLength(100)),
+
+        final BytesReference bytes = serialize(
+            true,
             handshakeCompatVersion,
+            true,
+            true,
             action,
             requestId,
-            true,
-            true
+            new TestRequest(randomAlphaOfLength(100))
         );
-
-        final BytesReference bytes = message.serialize(new BytesStreamOutput());
         int totalHeaderSize = TcpHeader.headerSize(handshakeCompatVersion) + bytes.getInt(TcpHeader.VARIABLE_HEADER_SIZE_POSITION);
 
         InboundDecoder decoder = new InboundDecoder(Version.CURRENT, PageCacheRecycler.NON_RECYCLING_INSTANCE);
@@ -294,19 +260,17 @@ public void testVersionIncompatibilityDecodeException() throws IOException {
         String action = "test-request";
         long requestId = randomNonNegativeLong();
         Version incompatibleVersion = Version.CURRENT.minimumCompatibilityVersion().minimumCompatibilityVersion();
-        OutboundMessage message = new OutboundMessage.Request(
-            threadContext,
-            new String[0],
-            new TestRequest(randomAlphaOfLength(100)),
+
+        final BytesReference bytes = serialize(
+            true,
             incompatibleVersion,
+            false,
+            true,
             action,
             requestId,
-            false,
-            true
+            new TestRequest(randomAlphaOfLength(100))
         );
 
-        final BytesReference bytes = message.serialize(new BytesStreamOutput());
-
         InboundDecoder decoder = new InboundDecoder(Version.CURRENT, PageCacheRecycler.NON_RECYCLING_INSTANCE);
         final ArrayList<Object> fragments = new ArrayList<>();
         final ReleasableBytesReference releasable1 = ReleasableBytesReference.wrap(bytes);
diff --git a/server/src/test/java/org/opensearch/transport/InboundHandlerTests.java b/server/src/test/java/org/opensearch/transport/InboundHandlerTests.java
index e002297911788..2553e7740990b 100644
--- a/server/src/test/java/org/opensearch/transport/InboundHandlerTests.java
+++ b/server/src/test/java/org/opensearch/transport/InboundHandlerTests.java
@@ -39,16 +39,17 @@
 import org.opensearch.Version;
 import org.opensearch.common.bytes.ReleasableBytesReference;
 import org.opensearch.common.collect.Tuple;
-import org.opensearch.common.io.stream.BytesStreamOutput;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.util.BigArrays;
+import org.opensearch.common.util.concurrent.ThreadContext;
 import org.opensearch.core.action.ActionListener;
 import org.opensearch.core.common.bytes.BytesArray;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.core.common.io.stream.InputStreamStreamInput;
 import org.opensearch.core.common.io.stream.NamedWriteableRegistry;
 import org.opensearch.core.common.io.stream.StreamInput;
+import org.opensearch.core.common.io.stream.Writeable;
 import org.opensearch.tasks.TaskManager;
 import org.opensearch.telemetry.tracing.noop.NoopTracer;
 import org.opensearch.test.MockLogAppender;
@@ -56,6 +57,7 @@
 import org.opensearch.test.VersionUtils;
 import org.opensearch.threadpool.TestThreadPool;
 import org.opensearch.threadpool.ThreadPool;
+import org.opensearch.transport.nativeprotocol.NativeInboundMessage;
 import org.junit.After;
 import org.junit.Before;
 
@@ -74,7 +76,17 @@
 import static org.hamcrest.CoreMatchers.startsWith;
 import static org.hamcrest.Matchers.instanceOf;
 
-public class InboundHandlerTests extends OpenSearchTestCase {
+public abstract class InboundHandlerTests extends OpenSearchTestCase {
+
+    public abstract BytesReference serializeOutboundRequest(
+        ThreadContext threadContext,
+        Writeable message,
+        Version version,
+        String action,
+        long requestId,
+        boolean compress,
+        boolean handshake
+    ) throws IOException;
 
     private final TestThreadPool threadPool = new TestThreadPool(getClass().getName());
     private final Version version = Version.CURRENT;
@@ -100,19 +112,17 @@ public void sendMessage(BytesReference reference, ActionListener<Void> listener)
         };
         NamedWriteableRegistry namedWriteableRegistry = new NamedWriteableRegistry(Collections.emptyList());
         TransportHandshaker handshaker = new TransportHandshaker(version, threadPool, (n, c, r, v) -> {});
-        outboundHandler = new OutboundHandler(
-            "node",
-            version,
-            new String[0],
-            new StatsTracker(),
-            threadPool,
-            BigArrays.NON_RECYCLING_INSTANCE
-        );
+        outboundHandler = new OutboundHandler(new StatsTracker(), threadPool);
         TransportKeepAlive keepAlive = new TransportKeepAlive(threadPool, outboundHandler::sendBytes);
         requestHandlers = new Transport.RequestHandlers();
         responseHandlers = new Transport.ResponseHandlers();
         handler = new InboundHandler(
+            "node",
+            version,
+            new String[0],
+            new StatsTracker(),
             threadPool,
+            BigArrays.NON_RECYCLING_INSTANCE,
             outboundHandler,
             namedWriteableRegistry,
             handshaker,
@@ -142,7 +152,7 @@ public void testPing() throws Exception {
         );
         requestHandlers.registerHandler(registry);
 
-        handler.inboundMessage(channel, new InboundMessage(null, true));
+        handler.inboundMessage(channel, new NativeInboundMessage(null, true));
         if (channel.isServerChannel()) {
             BytesReference ping = channel.getMessageCaptor().get();
             assertEquals('E', ping.get(0));
@@ -194,9 +204,9 @@ public TestResponse read(StreamInput in) throws IOException {
         );
         requestHandlers.registerHandler(registry);
         String requestValue = randomAlphaOfLength(10);
-        OutboundMessage.Request request = new OutboundMessage.Request(
+
+        BytesReference fullRequestBytes = serializeOutboundRequest(
             threadPool.getThreadContext(),
-            new String[0],
             new TestRequest(requestValue),
             version,
             action,
@@ -204,11 +214,13 @@ public TestResponse read(StreamInput in) throws IOException {
             false,
             false
         );
-
-        BytesReference fullRequestBytes = request.serialize(new BytesStreamOutput());
         BytesReference requestContent = fullRequestBytes.slice(headerSize, fullRequestBytes.length() - headerSize);
         Header requestHeader = new Header(fullRequestBytes.length() - 6, requestId, TransportStatus.setRequest((byte) 0), version);
-        InboundMessage requestMessage = new InboundMessage(requestHeader, ReleasableBytesReference.wrap(requestContent), () -> {});
+        NativeInboundMessage requestMessage = new NativeInboundMessage(
+            requestHeader,
+            ReleasableBytesReference.wrap(requestContent),
+            () -> {}
+        );
         requestHeader.finishParsingHeader(requestMessage.openOrGetStreamInput());
         handler.inboundMessage(channel, requestMessage);
 
@@ -229,7 +241,11 @@ public TestResponse read(StreamInput in) throws IOException {
         BytesReference fullResponseBytes = channel.getMessageCaptor().get();
         BytesReference responseContent = fullResponseBytes.slice(headerSize, fullResponseBytes.length() - headerSize);
         Header responseHeader = new Header(fullResponseBytes.length() - 6, requestId, responseStatus, version);
-        InboundMessage responseMessage = new InboundMessage(responseHeader, ReleasableBytesReference.wrap(responseContent), () -> {});
+        NativeInboundMessage responseMessage = new NativeInboundMessage(
+            responseHeader,
+            ReleasableBytesReference.wrap(responseContent),
+            () -> {}
+        );
         responseHeader.finishParsingHeader(responseMessage.openOrGetStreamInput());
         handler.inboundMessage(channel, responseMessage);
 
@@ -256,7 +272,7 @@ public void testSendsErrorResponseToHandshakeFromCompatibleVersion() throws Exce
             TransportStatus.setRequest(TransportStatus.setHandshake((byte) 0)),
             remoteVersion
         );
-        final InboundMessage requestMessage = unreadableInboundHandshake(remoteVersion, requestHeader);
+        final NativeInboundMessage requestMessage = unreadableInboundHandshake(remoteVersion, requestHeader);
         requestHeader.actionName = TransportHandshaker.HANDSHAKE_ACTION_NAME;
         requestHeader.headers = Tuple.tuple(Map.of(), Map.of());
         requestHeader.features = Set.of();
@@ -275,11 +291,11 @@ public void testClosesChannelOnErrorInHandshakeWithIncompatibleVersion() throws
         // response so we must just close the connection on an error. To avoid the failure disappearing into a black hole we at least log
         // it.
 
-        try (MockLogAppender mockAppender = MockLogAppender.createForLoggers(LogManager.getLogger(InboundHandler.class))) {
+        try (MockLogAppender mockAppender = MockLogAppender.createForLoggers(LogManager.getLogger(NativeMessageHandler.class))) {
             mockAppender.addExpectation(
                 new MockLogAppender.SeenEventExpectation(
                     "expected message",
-                    InboundHandler.class.getCanonicalName(),
+                    NativeMessageHandler.class.getCanonicalName(),
                     Level.WARN,
                     "could not send error response to handshake"
                 )
@@ -296,7 +312,7 @@ public void testClosesChannelOnErrorInHandshakeWithIncompatibleVersion() throws
                 TransportStatus.setRequest(TransportStatus.setHandshake((byte) 0)),
                 remoteVersion
             );
-            final InboundMessage requestMessage = unreadableInboundHandshake(remoteVersion, requestHeader);
+            final NativeInboundMessage requestMessage = unreadableInboundHandshake(remoteVersion, requestHeader);
             requestHeader.actionName = TransportHandshaker.HANDSHAKE_ACTION_NAME;
             requestHeader.headers = Tuple.tuple(Map.of(), Map.of());
             requestHeader.features = Set.of();
@@ -308,11 +324,11 @@ public void testClosesChannelOnErrorInHandshakeWithIncompatibleVersion() throws
     }
 
     public void testLogsSlowInboundProcessing() throws Exception {
-        try (MockLogAppender mockAppender = MockLogAppender.createForLoggers(LogManager.getLogger(InboundHandler.class))) {
+        try (MockLogAppender mockAppender = MockLogAppender.createForLoggers(LogManager.getLogger(NativeMessageHandler.class))) {
             mockAppender.addExpectation(
                 new MockLogAppender.SeenEventExpectation(
                     "expected message",
-                    InboundHandler.class.getCanonicalName(),
+                    NativeMessageHandler.class.getCanonicalName(),
                     Level.WARN,
                     "handling inbound transport message "
                 )
@@ -327,13 +343,17 @@ public void testLogsSlowInboundProcessing() throws Exception {
                 TransportStatus.setRequest(TransportStatus.setHandshake((byte) 0)),
                 remoteVersion
             );
-            final InboundMessage requestMessage = new InboundMessage(requestHeader, ReleasableBytesReference.wrap(BytesArray.EMPTY), () -> {
-                try {
-                    TimeUnit.SECONDS.sleep(1L);
-                } catch (InterruptedException e) {
-                    throw new AssertionError(e);
+            final NativeInboundMessage requestMessage = new NativeInboundMessage(
+                requestHeader,
+                ReleasableBytesReference.wrap(BytesArray.EMPTY),
+                () -> {
+                    try {
+                        TimeUnit.SECONDS.sleep(1L);
+                    } catch (InterruptedException e) {
+                        throw new AssertionError(e);
+                    }
                 }
-            });
+            );
             requestHeader.actionName = TransportHandshaker.HANDSHAKE_ACTION_NAME;
             requestHeader.headers = Tuple.tuple(Collections.emptyMap(), Collections.emptyMap());
             requestHeader.features = Set.of();
@@ -380,18 +400,8 @@ public TestResponse read(StreamInput in) throws IOException {
 
         requestHandlers.registerHandler(registry);
         String requestValue = randomAlphaOfLength(10);
-        OutboundMessage.Request request = new OutboundMessage.Request(
-            threadPool.getThreadContext(),
-            new String[0],
-            new TestRequest(requestValue),
-            version,
-            action,
-            requestId,
-            false,
-            false
-        );
 
-        outboundHandler.setMessageListener(new TransportMessageListener() {
+        handler.setMessageListener(new TransportMessageListener() {
             @Override
             public void onResponseSent(long requestId, String action, Exception error) {
                 exceptionCaptor.set(error);
@@ -399,7 +409,15 @@ public void onResponseSent(long requestId, String action, Exception error) {
         });
 
         // Create the request payload with 1 byte overflow
-        final BytesRef bytes = request.serialize(new BytesStreamOutput()).toBytesRef();
+        final BytesRef bytes = serializeOutboundRequest(
+            threadPool.getThreadContext(),
+            new TestRequest(requestValue),
+            version,
+            action,
+            requestId,
+            false,
+            false
+        ).toBytesRef();
         final ByteBuffer buffer = ByteBuffer.allocate(bytes.length + 1);
         buffer.put(bytes.bytes, 0, bytes.length);
         buffer.put((byte) 1);
@@ -407,7 +425,11 @@ public void onResponseSent(long requestId, String action, Exception error) {
         BytesReference fullRequestBytes = BytesReference.fromByteBuffer((ByteBuffer) buffer.flip());
         BytesReference requestContent = fullRequestBytes.slice(headerSize, fullRequestBytes.length() - headerSize);
         Header requestHeader = new Header(fullRequestBytes.length() - 6, requestId, TransportStatus.setRequest((byte) 0), version);
-        InboundMessage requestMessage = new InboundMessage(requestHeader, ReleasableBytesReference.wrap(requestContent), () -> {});
+        NativeInboundMessage requestMessage = new NativeInboundMessage(
+            requestHeader,
+            ReleasableBytesReference.wrap(requestContent),
+            () -> {}
+        );
         requestHeader.finishParsingHeader(requestMessage.openOrGetStreamInput());
         handler.inboundMessage(channel, requestMessage);
 
@@ -452,9 +474,16 @@ public TestResponse read(StreamInput in) throws IOException {
 
         requestHandlers.registerHandler(registry);
         String requestValue = randomAlphaOfLength(10);
-        OutboundMessage.Request request = new OutboundMessage.Request(
+
+        handler.setMessageListener(new TransportMessageListener() {
+            @Override
+            public void onResponseSent(long requestId, String action, Exception error) {
+                exceptionCaptor.set(error);
+            }
+        });
+
+        final BytesReference fullRequestBytes = serializeOutboundRequest(
             threadPool.getThreadContext(),
-            new String[0],
             new TestRequest(requestValue),
             version,
             action,
@@ -462,19 +491,14 @@ public TestResponse read(StreamInput in) throws IOException {
             false,
             false
         );
-
-        outboundHandler.setMessageListener(new TransportMessageListener() {
-            @Override
-            public void onResponseSent(long requestId, String action, Exception error) {
-                exceptionCaptor.set(error);
-            }
-        });
-
-        final BytesReference fullRequestBytes = request.serialize(new BytesStreamOutput());
         // Create the request payload by intentionally stripping 1 byte away
         BytesReference requestContent = fullRequestBytes.slice(headerSize, fullRequestBytes.length() - headerSize - 1);
         Header requestHeader = new Header(fullRequestBytes.length() - 6, requestId, TransportStatus.setRequest((byte) 0), version);
-        InboundMessage requestMessage = new InboundMessage(requestHeader, ReleasableBytesReference.wrap(requestContent), () -> {});
+        NativeInboundMessage requestMessage = new NativeInboundMessage(
+            requestHeader,
+            ReleasableBytesReference.wrap(requestContent),
+            () -> {}
+        );
         requestHeader.finishParsingHeader(requestMessage.openOrGetStreamInput());
         handler.inboundMessage(channel, requestMessage);
 
@@ -526,9 +550,9 @@ public TestResponse read(StreamInput in) throws IOException {
         );
         requestHandlers.registerHandler(registry);
         String requestValue = randomAlphaOfLength(10);
-        OutboundMessage.Request request = new OutboundMessage.Request(
+
+        BytesReference fullRequestBytes = serializeOutboundRequest(
             threadPool.getThreadContext(),
-            new String[0],
             new TestRequest(requestValue),
             version,
             action,
@@ -536,11 +560,13 @@ public TestResponse read(StreamInput in) throws IOException {
             false,
             false
         );
-
-        BytesReference fullRequestBytes = request.serialize(new BytesStreamOutput());
         BytesReference requestContent = fullRequestBytes.slice(headerSize, fullRequestBytes.length() - headerSize);
         Header requestHeader = new Header(fullRequestBytes.length() - 6, requestId, TransportStatus.setRequest((byte) 0), version);
-        InboundMessage requestMessage = new InboundMessage(requestHeader, ReleasableBytesReference.wrap(requestContent), () -> {});
+        NativeInboundMessage requestMessage = new NativeInboundMessage(
+            requestHeader,
+            ReleasableBytesReference.wrap(requestContent),
+            () -> {}
+        );
         requestHeader.finishParsingHeader(requestMessage.openOrGetStreamInput());
         handler.inboundMessage(channel, requestMessage);
 
@@ -562,7 +588,11 @@ public TestResponse read(StreamInput in) throws IOException {
         BytesReference fullResponseBytes = BytesReference.fromByteBuffer((ByteBuffer) buffer.flip());
         BytesReference responseContent = fullResponseBytes.slice(headerSize, fullResponseBytes.length() - headerSize);
         Header responseHeader = new Header(fullResponseBytes.length() - 6, requestId, responseStatus, version);
-        InboundMessage responseMessage = new InboundMessage(responseHeader, ReleasableBytesReference.wrap(responseContent), () -> {});
+        NativeInboundMessage responseMessage = new NativeInboundMessage(
+            responseHeader,
+            ReleasableBytesReference.wrap(responseContent),
+            () -> {}
+        );
         responseHeader.finishParsingHeader(responseMessage.openOrGetStreamInput());
         handler.inboundMessage(channel, responseMessage);
 
@@ -614,9 +644,9 @@ public TestResponse read(StreamInput in) throws IOException {
         );
         requestHandlers.registerHandler(registry);
         String requestValue = randomAlphaOfLength(10);
-        OutboundMessage.Request request = new OutboundMessage.Request(
+
+        BytesReference fullRequestBytes = serializeOutboundRequest(
             threadPool.getThreadContext(),
-            new String[0],
             new TestRequest(requestValue),
             version,
             action,
@@ -624,11 +654,13 @@ public TestResponse read(StreamInput in) throws IOException {
             false,
             false
         );
-
-        BytesReference fullRequestBytes = request.serialize(new BytesStreamOutput());
         BytesReference requestContent = fullRequestBytes.slice(headerSize, fullRequestBytes.length() - headerSize);
         Header requestHeader = new Header(fullRequestBytes.length() - 6, requestId, TransportStatus.setRequest((byte) 0), version);
-        InboundMessage requestMessage = new InboundMessage(requestHeader, ReleasableBytesReference.wrap(requestContent), () -> {});
+        NativeInboundMessage requestMessage = new NativeInboundMessage(
+            requestHeader,
+            ReleasableBytesReference.wrap(requestContent),
+            () -> {}
+        );
         requestHeader.finishParsingHeader(requestMessage.openOrGetStreamInput());
         handler.inboundMessage(channel, requestMessage);
 
@@ -645,7 +677,11 @@ public TestResponse read(StreamInput in) throws IOException {
         // Create the response payload by intentionally stripping 1 byte away
         BytesReference responseContent = fullResponseBytes.slice(headerSize, fullResponseBytes.length() - headerSize - 1);
         Header responseHeader = new Header(fullResponseBytes.length() - 6, requestId, responseStatus, version);
-        InboundMessage responseMessage = new InboundMessage(responseHeader, ReleasableBytesReference.wrap(responseContent), () -> {});
+        NativeInboundMessage responseMessage = new NativeInboundMessage(
+            responseHeader,
+            ReleasableBytesReference.wrap(responseContent),
+            () -> {}
+        );
         responseHeader.finishParsingHeader(responseMessage.openOrGetStreamInput());
         handler.inboundMessage(channel, responseMessage);
 
@@ -654,8 +690,8 @@ public TestResponse read(StreamInput in) throws IOException {
         assertThat(exceptionCaptor.get().getMessage(), containsString("Failed to deserialize response from handler"));
     }
 
-    private static InboundMessage unreadableInboundHandshake(Version remoteVersion, Header requestHeader) {
-        return new InboundMessage(requestHeader, ReleasableBytesReference.wrap(BytesArray.EMPTY), () -> {}) {
+    private static NativeInboundMessage unreadableInboundHandshake(Version remoteVersion, Header requestHeader) {
+        return new NativeInboundMessage(requestHeader, ReleasableBytesReference.wrap(BytesArray.EMPTY), () -> {}) {
             @Override
             public StreamInput openOrGetStreamInput() {
                 final StreamInput streamInput = new InputStreamStreamInput(new InputStream() {
diff --git a/server/src/test/java/org/opensearch/transport/InboundPipelineTests.java b/server/src/test/java/org/opensearch/transport/InboundPipelineTests.java
index ae4b537223394..5a89bf1e0ead3 100644
--- a/server/src/test/java/org/opensearch/transport/InboundPipelineTests.java
+++ b/server/src/test/java/org/opensearch/transport/InboundPipelineTests.java
@@ -49,10 +49,10 @@
 import org.opensearch.core.common.bytes.BytesArray;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.test.OpenSearchTestCase;
+import org.opensearch.transport.nativeprotocol.NativeInboundMessage;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 import java.util.concurrent.atomic.AtomicBoolean;
@@ -63,33 +63,44 @@
 
 import static org.hamcrest.Matchers.instanceOf;
 
-public class InboundPipelineTests extends OpenSearchTestCase {
+public abstract class InboundPipelineTests extends OpenSearchTestCase {
 
     private static final int BYTE_THRESHOLD = 128 * 1024;
-    private final ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
-
-    public void testPipelineHandling() throws IOException {
+    public final ThreadContext threadContext = new ThreadContext(Settings.EMPTY);
+
+    protected abstract BytesReference serialize(
+        boolean isRequest,
+        Version version,
+        boolean handshake,
+        boolean compress,
+        String action,
+        long requestId,
+        String value
+    ) throws IOException;
+
+    public void testPipelineHandlingForNativeProtocol() throws IOException {
         final List<Tuple<MessageData, Exception>> expected = new ArrayList<>();
         final List<Tuple<MessageData, Exception>> actual = new ArrayList<>();
         final List<ReleasableBytesReference> toRelease = new ArrayList<>();
-        final BiConsumer<TcpChannel, InboundMessage> messageHandler = (c, m) -> {
+        final BiConsumer<TcpChannel, ProtocolInboundMessage> messageHandler = (c, m) -> {
             try {
-                final Header header = m.getHeader();
+                NativeInboundMessage message = (NativeInboundMessage) m;
+                final Header header = message.getHeader();
                 final MessageData actualData;
                 final Version version = header.getVersion();
                 final boolean isRequest = header.isRequest();
                 final long requestId = header.getRequestId();
                 final boolean isCompressed = header.isCompressed();
-                if (m.isShortCircuit()) {
+                if (message.isShortCircuit()) {
                     actualData = new MessageData(version, requestId, isRequest, isCompressed, header.getActionName(), null);
                 } else if (isRequest) {
-                    final TestRequest request = new TestRequest(m.openOrGetStreamInput());
-                    actualData = new MessageData(version, requestId, isRequest, isCompressed, header.getActionName(), request.value);
+                    final TestRequest request = new TestRequest(message.openOrGetStreamInput());
+                    actualData = new MessageData(version, requestId, isRequest, isCompressed, header.getActionName(), request.getValue());
                 } else {
-                    final TestResponse response = new TestResponse(m.openOrGetStreamInput());
-                    actualData = new MessageData(version, requestId, isRequest, isCompressed, null, response.value);
+                    final TestResponse response = new TestResponse(message.openOrGetStreamInput());
+                    actualData = new MessageData(version, requestId, isRequest, isCompressed, null, response.getValue());
                 }
-                actual.add(new Tuple<>(actualData, m.getException()));
+                actual.add(new Tuple<>(actualData, message.getException()));
             } catch (IOException e) {
                 throw new AssertionError(e);
             }
@@ -126,49 +137,23 @@ public void testPipelineHandling() throws IOException {
                     final MessageData messageData;
                     Exception expectedExceptionClass = null;
 
-                    OutboundMessage message;
+                    // NativeOutboundMessage message;
+                    final BytesReference reference;
                     if (isRequest) {
                         if (rarely()) {
                             messageData = new MessageData(version, requestId, true, isCompressed, breakThisAction, null);
-                            message = new OutboundMessage.Request(
-                                threadContext,
-                                new String[0],
-                                new TestRequest(value),
-                                version,
-                                breakThisAction,
-                                requestId,
-                                false,
-                                isCompressed
-                            );
+                            reference = serialize(true, version, false, isCompressed, breakThisAction, requestId, value);
                             expectedExceptionClass = new CircuitBreakingException("", CircuitBreaker.Durability.PERMANENT);
                         } else {
                             messageData = new MessageData(version, requestId, true, isCompressed, actionName, value);
-                            message = new OutboundMessage.Request(
-                                threadContext,
-                                new String[0],
-                                new TestRequest(value),
-                                version,
-                                actionName,
-                                requestId,
-                                false,
-                                isCompressed
-                            );
+                            reference = serialize(true, version, false, isCompressed, actionName, requestId, value);
                         }
                     } else {
                         messageData = new MessageData(version, requestId, false, isCompressed, null, value);
-                        message = new OutboundMessage.Response(
-                            threadContext,
-                            Collections.emptySet(),
-                            new TestResponse(value),
-                            version,
-                            requestId,
-                            false,
-                            isCompressed
-                        );
+                        reference = serialize(false, version, false, isCompressed, actionName, requestId, value);
                     }
 
                     expected.add(new Tuple<>(messageData, expectedExceptionClass));
-                    final BytesReference reference = message.serialize(new BytesStreamOutput());
                     Streams.copy(reference.streamInput(), streamOutput);
                 }
 
@@ -214,7 +199,7 @@ public void testPipelineHandling() throws IOException {
     }
 
     public void testDecodeExceptionIsPropagated() throws IOException {
-        BiConsumer<TcpChannel, InboundMessage> messageHandler = (c, m) -> {};
+        BiConsumer<TcpChannel, ProtocolInboundMessage> messageHandler = (c, m) -> {};
         final StatsTracker statsTracker = new StatsTracker();
         final LongSupplier millisSupplier = () -> TimeValue.nsecToMSec(System.nanoTime());
         final InboundDecoder decoder = new InboundDecoder(Version.CURRENT, PageCacheRecycler.NON_RECYCLING_INSTANCE);
@@ -229,31 +214,7 @@ public void testDecodeExceptionIsPropagated() throws IOException {
             final boolean isRequest = randomBoolean();
             final long requestId = randomNonNegativeLong();
 
-            OutboundMessage message;
-            if (isRequest) {
-                message = new OutboundMessage.Request(
-                    threadContext,
-                    new String[0],
-                    new TestRequest(value),
-                    invalidVersion,
-                    actionName,
-                    requestId,
-                    false,
-                    false
-                );
-            } else {
-                message = new OutboundMessage.Response(
-                    threadContext,
-                    Collections.emptySet(),
-                    new TestResponse(value),
-                    invalidVersion,
-                    requestId,
-                    false,
-                    false
-                );
-            }
-
-            final BytesReference reference = message.serialize(streamOutput);
+            final BytesReference reference = serialize(isRequest, invalidVersion, false, false, actionName, requestId, value);
             try (ReleasableBytesReference releasable = ReleasableBytesReference.wrap(reference)) {
                 expectThrows(IllegalStateException.class, () -> pipeline.handleBytes(new FakeTcpChannel(), releasable));
             }
@@ -268,7 +229,7 @@ public void testDecodeExceptionIsPropagated() throws IOException {
     }
 
     public void testEnsureBodyIsNotPrematurelyReleased() throws IOException {
-        BiConsumer<TcpChannel, InboundMessage> messageHandler = (c, m) -> {};
+        BiConsumer<TcpChannel, ProtocolInboundMessage> messageHandler = (c, m) -> {};
         final StatsTracker statsTracker = new StatsTracker();
         final LongSupplier millisSupplier = () -> TimeValue.nsecToMSec(System.nanoTime());
         final InboundDecoder decoder = new InboundDecoder(Version.CURRENT, PageCacheRecycler.NON_RECYCLING_INSTANCE);
@@ -283,31 +244,7 @@ public void testEnsureBodyIsNotPrematurelyReleased() throws IOException {
             final boolean isRequest = randomBoolean();
             final long requestId = randomNonNegativeLong();
 
-            OutboundMessage message;
-            if (isRequest) {
-                message = new OutboundMessage.Request(
-                    threadContext,
-                    new String[0],
-                    new TestRequest(value),
-                    version,
-                    actionName,
-                    requestId,
-                    false,
-                    false
-                );
-            } else {
-                message = new OutboundMessage.Response(
-                    threadContext,
-                    Collections.emptySet(),
-                    new TestResponse(value),
-                    version,
-                    requestId,
-                    false,
-                    false
-                );
-            }
-
-            final BytesReference reference = message.serialize(streamOutput);
+            final BytesReference reference = serialize(isRequest, version, false, false, actionName, requestId, value);
             final int fixedHeaderSize = TcpHeader.headerSize(Version.CURRENT);
             final int variableHeaderSize = reference.getInt(fixedHeaderSize - 4);
             final int totalHeaderSize = fixedHeaderSize + variableHeaderSize;
diff --git a/server/src/test/java/org/opensearch/transport/NativeOutboundHandlerTests.java b/server/src/test/java/org/opensearch/transport/NativeOutboundHandlerTests.java
new file mode 100644
index 0000000000000..01f19bea7a37f
--- /dev/null
+++ b/server/src/test/java/org/opensearch/transport/NativeOutboundHandlerTests.java
@@ -0,0 +1,302 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+package org.opensearch.transport;
+
+import org.opensearch.OpenSearchException;
+import org.opensearch.Version;
+import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.common.bytes.ReleasableBytesReference;
+import org.opensearch.common.collect.Tuple;
+import org.opensearch.common.io.stream.BytesStreamOutput;
+import org.opensearch.common.unit.TimeValue;
+import org.opensearch.common.util.BigArrays;
+import org.opensearch.common.util.PageCacheRecycler;
+import org.opensearch.common.util.concurrent.ThreadContext;
+import org.opensearch.common.util.io.Streams;
+import org.opensearch.core.action.ActionListener;
+import org.opensearch.core.common.breaker.CircuitBreaker;
+import org.opensearch.core.common.breaker.NoopCircuitBreaker;
+import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.core.common.transport.TransportAddress;
+import org.opensearch.core.transport.TransportResponse;
+import org.opensearch.test.OpenSearchTestCase;
+import org.opensearch.threadpool.TestThreadPool;
+import org.opensearch.threadpool.ThreadPool;
+import org.opensearch.transport.nativeprotocol.NativeInboundMessage;
+import org.opensearch.transport.nativeprotocol.NativeOutboundHandler;
+import org.junit.After;
+import org.junit.Before;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.LongSupplier;
+import java.util.function.Predicate;
+import java.util.function.Supplier;
+
+import static org.hamcrest.Matchers.instanceOf;
+
+public class NativeOutboundHandlerTests extends OpenSearchTestCase {
+
+    private final String feature1 = "feature1";
+    private final String feature2 = "feature2";
+    private final TestThreadPool threadPool = new TestThreadPool(getClass().getName());
+    private final TransportRequestOptions options = TransportRequestOptions.EMPTY;
+    private final AtomicReference<Tuple<Header, BytesReference>> message = new AtomicReference<>();
+    private InboundPipeline pipeline;
+    private OutboundHandler handler;
+    private NativeOutboundHandler nativeOutboundHandler;
+    private FakeTcpChannel channel;
+    private DiscoveryNode node;
+
+    @Before
+    public void setUp() throws Exception {
+        super.setUp();
+        channel = new FakeTcpChannel(randomBoolean(), buildNewFakeTransportAddress().address(), buildNewFakeTransportAddress().address());
+        TransportAddress transportAddress = buildNewFakeTransportAddress();
+        node = new DiscoveryNode("", transportAddress, Version.CURRENT);
+        String[] features = { feature1, feature2 };
+        StatsTracker statsTracker = new StatsTracker();
+        handler = new OutboundHandler(statsTracker, threadPool);
+        nativeOutboundHandler = new NativeOutboundHandler(
+            "node",
+            Version.CURRENT,
+            features,
+            statsTracker,
+            threadPool,
+            BigArrays.NON_RECYCLING_INSTANCE,
+            handler
+        );
+
+        final LongSupplier millisSupplier = () -> TimeValue.nsecToMSec(System.nanoTime());
+        final InboundDecoder decoder = new InboundDecoder(Version.CURRENT, PageCacheRecycler.NON_RECYCLING_INSTANCE);
+        final Supplier<CircuitBreaker> breaker = () -> new NoopCircuitBreaker("test");
+        final InboundAggregator aggregator = new InboundAggregator(breaker, (Predicate<String>) action -> true);
+        pipeline = new InboundPipeline(statsTracker, millisSupplier, decoder, aggregator, (c, m) -> {
+            try (BytesStreamOutput streamOutput = new BytesStreamOutput()) {
+                NativeInboundMessage m1 = (NativeInboundMessage) m;
+                Streams.copy(m1.openOrGetStreamInput(), streamOutput);
+                message.set(new Tuple<>(m1.getHeader(), streamOutput.bytes()));
+            } catch (IOException e) {
+                throw new AssertionError(e);
+            }
+        });
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        ThreadPool.terminate(threadPool, 10, TimeUnit.SECONDS);
+        super.tearDown();
+    }
+
+    public void testSendRequest() throws IOException {
+        ThreadContext threadContext = threadPool.getThreadContext();
+        Version version = randomFrom(Version.CURRENT, Version.CURRENT.minimumCompatibilityVersion());
+        String action = "handshake";
+        long requestId = randomLongBetween(0, 300);
+        boolean isHandshake = randomBoolean();
+        boolean compress = randomBoolean();
+        String value = "message";
+        threadContext.putHeader("header", "header_value");
+        TestRequest request = new TestRequest(value);
+
+        AtomicReference<DiscoveryNode> nodeRef = new AtomicReference<>();
+        AtomicLong requestIdRef = new AtomicLong();
+        AtomicReference<String> actionRef = new AtomicReference<>();
+        AtomicReference<TransportRequest> requestRef = new AtomicReference<>();
+        nativeOutboundHandler.setMessageListener(new TransportMessageListener() {
+            @Override
+            public void onRequestSent(
+                DiscoveryNode node,
+                long requestId,
+                String action,
+                TransportRequest request,
+                TransportRequestOptions options
+            ) {
+                nodeRef.set(node);
+                requestIdRef.set(requestId);
+                actionRef.set(action);
+                requestRef.set(request);
+            }
+        });
+        nativeOutboundHandler.sendRequest(node, channel, requestId, action, request, options, version, compress, isHandshake);
+
+        BytesReference reference = channel.getMessageCaptor().get();
+        ActionListener<Void> sendListener = channel.getListenerCaptor().get();
+        if (randomBoolean()) {
+            sendListener.onResponse(null);
+        } else {
+            sendListener.onFailure(new IOException("failed"));
+        }
+        assertEquals(node, nodeRef.get());
+        assertEquals(requestId, requestIdRef.get());
+        assertEquals(action, actionRef.get());
+        assertEquals(request, requestRef.get());
+
+        pipeline.handleBytes(channel, new ReleasableBytesReference(reference, () -> {}));
+        final Tuple<Header, BytesReference> tuple = message.get();
+        final Header header = tuple.v1();
+        final TestRequest message = new TestRequest(tuple.v2().streamInput());
+        assertEquals(version, header.getVersion());
+        assertEquals(requestId, header.getRequestId());
+        assertTrue(header.isRequest());
+        assertFalse(header.isResponse());
+        if (isHandshake) {
+            assertTrue(header.isHandshake());
+        } else {
+            assertFalse(header.isHandshake());
+        }
+        if (compress) {
+            assertTrue(header.isCompressed());
+        } else {
+            assertFalse(header.isCompressed());
+        }
+
+        assertEquals(value, message.getValue());
+        assertEquals("header_value", header.getHeaders().v1().get("header"));
+    }
+
+    public void testSendResponse() throws IOException {
+        ThreadContext threadContext = threadPool.getThreadContext();
+        Version version = randomFrom(Version.CURRENT, Version.CURRENT.minimumCompatibilityVersion());
+        String action = "handshake";
+        long requestId = randomLongBetween(0, 300);
+        boolean isHandshake = randomBoolean();
+        boolean compress = randomBoolean();
+        String value = "message";
+        threadContext.putHeader("header", "header_value");
+        TestResponse response = new TestResponse(value);
+
+        AtomicLong requestIdRef = new AtomicLong();
+        AtomicReference<String> actionRef = new AtomicReference<>();
+        AtomicReference<TransportResponse> responseRef = new AtomicReference<>();
+        nativeOutboundHandler.setMessageListener(new TransportMessageListener() {
+            @Override
+            public void onResponseSent(long requestId, String action, TransportResponse response) {
+                requestIdRef.set(requestId);
+                actionRef.set(action);
+                responseRef.set(response);
+            }
+        });
+        nativeOutboundHandler.sendResponse(version, Collections.emptySet(), channel, requestId, action, response, compress, isHandshake);
+
+        BytesReference reference = channel.getMessageCaptor().get();
+        ActionListener<Void> sendListener = channel.getListenerCaptor().get();
+        if (randomBoolean()) {
+            sendListener.onResponse(null);
+        } else {
+            sendListener.onFailure(new IOException("failed"));
+        }
+        assertEquals(requestId, requestIdRef.get());
+        assertEquals(action, actionRef.get());
+        assertEquals(response, responseRef.get());
+
+        pipeline.handleBytes(channel, new ReleasableBytesReference(reference, () -> {}));
+        final Tuple<Header, BytesReference> tuple = message.get();
+        final Header header = tuple.v1();
+        final TestResponse message = new TestResponse(tuple.v2().streamInput());
+        assertEquals(version, header.getVersion());
+        assertEquals(requestId, header.getRequestId());
+        assertFalse(header.isRequest());
+        assertTrue(header.isResponse());
+        if (isHandshake) {
+            assertTrue(header.isHandshake());
+        } else {
+            assertFalse(header.isHandshake());
+        }
+        if (compress) {
+            assertTrue(header.isCompressed());
+        } else {
+            assertFalse(header.isCompressed());
+        }
+
+        assertFalse(header.isError());
+
+        assertEquals(value, message.getValue());
+        assertEquals("header_value", header.getHeaders().v1().get("header"));
+    }
+
+    public void testErrorResponse() throws IOException {
+        ThreadContext threadContext = threadPool.getThreadContext();
+        Version version = randomFrom(Version.CURRENT, Version.CURRENT.minimumCompatibilityVersion());
+        String action = "handshake";
+        long requestId = randomLongBetween(0, 300);
+        threadContext.putHeader("header", "header_value");
+        OpenSearchException error = new OpenSearchException("boom");
+
+        AtomicLong requestIdRef = new AtomicLong();
+        AtomicReference<String> actionRef = new AtomicReference<>();
+        AtomicReference<Exception> responseRef = new AtomicReference<>();
+        nativeOutboundHandler.setMessageListener(new TransportMessageListener() {
+            @Override
+            public void onResponseSent(long requestId, String action, Exception error) {
+                requestIdRef.set(requestId);
+                actionRef.set(action);
+                responseRef.set(error);
+            }
+        });
+        nativeOutboundHandler.sendErrorResponse(version, Collections.emptySet(), channel, requestId, action, error);
+
+        BytesReference reference = channel.getMessageCaptor().get();
+        ActionListener<Void> sendListener = channel.getListenerCaptor().get();
+        if (randomBoolean()) {
+            sendListener.onResponse(null);
+        } else {
+            sendListener.onFailure(new IOException("failed"));
+        }
+        assertEquals(requestId, requestIdRef.get());
+        assertEquals(action, actionRef.get());
+        assertEquals(error, responseRef.get());
+
+        pipeline.handleBytes(channel, new ReleasableBytesReference(reference, () -> {}));
+        final Tuple<Header, BytesReference> tuple = message.get();
+        final Header header = tuple.v1();
+        assertEquals(version, header.getVersion());
+        assertEquals(requestId, header.getRequestId());
+        assertFalse(header.isRequest());
+        assertTrue(header.isResponse());
+        assertFalse(header.isCompressed());
+        assertFalse(header.isHandshake());
+        assertTrue(header.isError());
+
+        RemoteTransportException remoteException = tuple.v2().streamInput().readException();
+        assertThat(remoteException.getCause(), instanceOf(OpenSearchException.class));
+        assertEquals(remoteException.getCause().getMessage(), "boom");
+        assertEquals(action, remoteException.action());
+        assertEquals(channel.getLocalAddress(), remoteException.address().address());
+
+        assertEquals("header_value", header.getHeaders().v1().get("header"));
+    }
+}
diff --git a/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java b/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java
index ff99435f765d8..7e7c60e2d3d29 100644
--- a/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java
+++ b/server/src/test/java/org/opensearch/transport/OutboundHandlerTests.java
@@ -32,24 +32,9 @@
 
 package org.opensearch.transport;
 
-import org.opensearch.OpenSearchException;
-import org.opensearch.Version;
-import org.opensearch.cluster.node.DiscoveryNode;
-import org.opensearch.common.bytes.ReleasableBytesReference;
-import org.opensearch.common.collect.Tuple;
-import org.opensearch.common.io.stream.BytesStreamOutput;
-import org.opensearch.common.unit.TimeValue;
-import org.opensearch.common.util.BigArrays;
-import org.opensearch.common.util.PageCacheRecycler;
-import org.opensearch.common.util.concurrent.ThreadContext;
-import org.opensearch.common.util.io.Streams;
 import org.opensearch.core.action.ActionListener;
-import org.opensearch.core.common.breaker.CircuitBreaker;
-import org.opensearch.core.common.breaker.NoopCircuitBreaker;
 import org.opensearch.core.common.bytes.BytesArray;
 import org.opensearch.core.common.bytes.BytesReference;
-import org.opensearch.core.common.transport.TransportAddress;
-import org.opensearch.core.transport.TransportResponse;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.threadpool.TestThreadPool;
 import org.opensearch.threadpool.ThreadPool;
@@ -58,51 +43,22 @@
 
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
-import java.util.Collections;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicLong;
 import java.util.concurrent.atomic.AtomicReference;
-import java.util.function.LongSupplier;
-import java.util.function.Predicate;
-import java.util.function.Supplier;
-
-import static org.hamcrest.Matchers.instanceOf;
 
 public class OutboundHandlerTests extends OpenSearchTestCase {
 
-    private final String feature1 = "feature1";
-    private final String feature2 = "feature2";
     private final TestThreadPool threadPool = new TestThreadPool(getClass().getName());
-    private final TransportRequestOptions options = TransportRequestOptions.EMPTY;
-    private final AtomicReference<Tuple<Header, BytesReference>> message = new AtomicReference<>();
-    private InboundPipeline pipeline;
     private OutboundHandler handler;
     private FakeTcpChannel channel;
-    private DiscoveryNode node;
 
     @Before
     public void setUp() throws Exception {
         super.setUp();
         channel = new FakeTcpChannel(randomBoolean(), buildNewFakeTransportAddress().address(), buildNewFakeTransportAddress().address());
-        TransportAddress transportAddress = buildNewFakeTransportAddress();
-        node = new DiscoveryNode("", transportAddress, Version.CURRENT);
-        String[] features = { feature1, feature2 };
         StatsTracker statsTracker = new StatsTracker();
-        handler = new OutboundHandler("node", Version.CURRENT, features, statsTracker, threadPool, BigArrays.NON_RECYCLING_INSTANCE);
-
-        final LongSupplier millisSupplier = () -> TimeValue.nsecToMSec(System.nanoTime());
-        final InboundDecoder decoder = new InboundDecoder(Version.CURRENT, PageCacheRecycler.NON_RECYCLING_INSTANCE);
-        final Supplier<CircuitBreaker> breaker = () -> new NoopCircuitBreaker("test");
-        final InboundAggregator aggregator = new InboundAggregator(breaker, (Predicate<String>) action -> true);
-        pipeline = new InboundPipeline(statsTracker, millisSupplier, decoder, aggregator, (c, m) -> {
-            try (BytesStreamOutput streamOutput = new BytesStreamOutput()) {
-                Streams.copy(m.openOrGetStreamInput(), streamOutput);
-                message.set(new Tuple<>(m.getHeader(), streamOutput.bytes()));
-            } catch (IOException e) {
-                throw new AssertionError(e);
-            }
-        });
+        handler = new OutboundHandler(statsTracker, threadPool);
     }
 
     @After
@@ -135,182 +91,4 @@ public void testSendRawBytes() {
         assertEquals(bytesArray, reference);
     }
 
-    public void testSendRequest() throws IOException {
-        ThreadContext threadContext = threadPool.getThreadContext();
-        Version version = randomFrom(Version.CURRENT, Version.CURRENT.minimumCompatibilityVersion());
-        String action = "handshake";
-        long requestId = randomLongBetween(0, 300);
-        boolean isHandshake = randomBoolean();
-        boolean compress = randomBoolean();
-        String value = "message";
-        threadContext.putHeader("header", "header_value");
-        TestRequest request = new TestRequest(value);
-
-        AtomicReference<DiscoveryNode> nodeRef = new AtomicReference<>();
-        AtomicLong requestIdRef = new AtomicLong();
-        AtomicReference<String> actionRef = new AtomicReference<>();
-        AtomicReference<TransportRequest> requestRef = new AtomicReference<>();
-        handler.setMessageListener(new TransportMessageListener() {
-            @Override
-            public void onRequestSent(
-                DiscoveryNode node,
-                long requestId,
-                String action,
-                TransportRequest request,
-                TransportRequestOptions options
-            ) {
-                nodeRef.set(node);
-                requestIdRef.set(requestId);
-                actionRef.set(action);
-                requestRef.set(request);
-            }
-        });
-        handler.sendRequest(node, channel, requestId, action, request, options, version, compress, isHandshake);
-
-        BytesReference reference = channel.getMessageCaptor().get();
-        ActionListener<Void> sendListener = channel.getListenerCaptor().get();
-        if (randomBoolean()) {
-            sendListener.onResponse(null);
-        } else {
-            sendListener.onFailure(new IOException("failed"));
-        }
-        assertEquals(node, nodeRef.get());
-        assertEquals(requestId, requestIdRef.get());
-        assertEquals(action, actionRef.get());
-        assertEquals(request, requestRef.get());
-
-        pipeline.handleBytes(channel, new ReleasableBytesReference(reference, () -> {}));
-        final Tuple<Header, BytesReference> tuple = message.get();
-        final Header header = tuple.v1();
-        final TestRequest message = new TestRequest(tuple.v2().streamInput());
-        assertEquals(version, header.getVersion());
-        assertEquals(requestId, header.getRequestId());
-        assertTrue(header.isRequest());
-        assertFalse(header.isResponse());
-        if (isHandshake) {
-            assertTrue(header.isHandshake());
-        } else {
-            assertFalse(header.isHandshake());
-        }
-        if (compress) {
-            assertTrue(header.isCompressed());
-        } else {
-            assertFalse(header.isCompressed());
-        }
-
-        assertEquals(value, message.value);
-        assertEquals("header_value", header.getHeaders().v1().get("header"));
-    }
-
-    public void testSendResponse() throws IOException {
-        ThreadContext threadContext = threadPool.getThreadContext();
-        Version version = randomFrom(Version.CURRENT, Version.CURRENT.minimumCompatibilityVersion());
-        String action = "handshake";
-        long requestId = randomLongBetween(0, 300);
-        boolean isHandshake = randomBoolean();
-        boolean compress = randomBoolean();
-        String value = "message";
-        threadContext.putHeader("header", "header_value");
-        TestResponse response = new TestResponse(value);
-
-        AtomicLong requestIdRef = new AtomicLong();
-        AtomicReference<String> actionRef = new AtomicReference<>();
-        AtomicReference<TransportResponse> responseRef = new AtomicReference<>();
-        handler.setMessageListener(new TransportMessageListener() {
-            @Override
-            public void onResponseSent(long requestId, String action, TransportResponse response) {
-                requestIdRef.set(requestId);
-                actionRef.set(action);
-                responseRef.set(response);
-            }
-        });
-        handler.sendResponse(version, Collections.emptySet(), channel, requestId, action, response, compress, isHandshake);
-
-        BytesReference reference = channel.getMessageCaptor().get();
-        ActionListener<Void> sendListener = channel.getListenerCaptor().get();
-        if (randomBoolean()) {
-            sendListener.onResponse(null);
-        } else {
-            sendListener.onFailure(new IOException("failed"));
-        }
-        assertEquals(requestId, requestIdRef.get());
-        assertEquals(action, actionRef.get());
-        assertEquals(response, responseRef.get());
-
-        pipeline.handleBytes(channel, new ReleasableBytesReference(reference, () -> {}));
-        final Tuple<Header, BytesReference> tuple = message.get();
-        final Header header = tuple.v1();
-        final TestResponse message = new TestResponse(tuple.v2().streamInput());
-        assertEquals(version, header.getVersion());
-        assertEquals(requestId, header.getRequestId());
-        assertFalse(header.isRequest());
-        assertTrue(header.isResponse());
-        if (isHandshake) {
-            assertTrue(header.isHandshake());
-        } else {
-            assertFalse(header.isHandshake());
-        }
-        if (compress) {
-            assertTrue(header.isCompressed());
-        } else {
-            assertFalse(header.isCompressed());
-        }
-
-        assertFalse(header.isError());
-
-        assertEquals(value, message.value);
-        assertEquals("header_value", header.getHeaders().v1().get("header"));
-    }
-
-    public void testErrorResponse() throws IOException {
-        ThreadContext threadContext = threadPool.getThreadContext();
-        Version version = randomFrom(Version.CURRENT, Version.CURRENT.minimumCompatibilityVersion());
-        String action = "handshake";
-        long requestId = randomLongBetween(0, 300);
-        threadContext.putHeader("header", "header_value");
-        OpenSearchException error = new OpenSearchException("boom");
-
-        AtomicLong requestIdRef = new AtomicLong();
-        AtomicReference<String> actionRef = new AtomicReference<>();
-        AtomicReference<Exception> responseRef = new AtomicReference<>();
-        handler.setMessageListener(new TransportMessageListener() {
-            @Override
-            public void onResponseSent(long requestId, String action, Exception error) {
-                requestIdRef.set(requestId);
-                actionRef.set(action);
-                responseRef.set(error);
-            }
-        });
-        handler.sendErrorResponse(version, Collections.emptySet(), channel, requestId, action, error);
-
-        BytesReference reference = channel.getMessageCaptor().get();
-        ActionListener<Void> sendListener = channel.getListenerCaptor().get();
-        if (randomBoolean()) {
-            sendListener.onResponse(null);
-        } else {
-            sendListener.onFailure(new IOException("failed"));
-        }
-        assertEquals(requestId, requestIdRef.get());
-        assertEquals(action, actionRef.get());
-        assertEquals(error, responseRef.get());
-
-        pipeline.handleBytes(channel, new ReleasableBytesReference(reference, () -> {}));
-        final Tuple<Header, BytesReference> tuple = message.get();
-        final Header header = tuple.v1();
-        assertEquals(version, header.getVersion());
-        assertEquals(requestId, header.getRequestId());
-        assertFalse(header.isRequest());
-        assertTrue(header.isResponse());
-        assertFalse(header.isCompressed());
-        assertFalse(header.isHandshake());
-        assertTrue(header.isError());
-
-        RemoteTransportException remoteException = tuple.v2().streamInput().readException();
-        assertThat(remoteException.getCause(), instanceOf(OpenSearchException.class));
-        assertEquals(remoteException.getCause().getMessage(), "boom");
-        assertEquals(action, remoteException.action());
-        assertEquals(channel.getLocalAddress(), remoteException.address().address());
-
-        assertEquals("header_value", header.getHeaders().v1().get("header"));
-    }
 }
diff --git a/server/src/test/java/org/opensearch/transport/TcpTransportTests.java b/server/src/test/java/org/opensearch/transport/TcpTransportTests.java
index 7ab78cca7d615..7c5c9ec12360d 100644
--- a/server/src/test/java/org/opensearch/transport/TcpTransportTests.java
+++ b/server/src/test/java/org/opensearch/transport/TcpTransportTests.java
@@ -43,7 +43,6 @@
 import org.opensearch.common.network.NetworkService;
 import org.opensearch.common.network.NetworkUtils;
 import org.opensearch.common.settings.Settings;
-import org.opensearch.common.util.BigArrays;
 import org.opensearch.common.util.MockPageCacheRecycler;
 import org.opensearch.core.common.transport.TransportAddress;
 import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
@@ -540,19 +539,7 @@ private void testExceptionHandling(
             final PlainActionFuture<Void> listener = new PlainActionFuture<>();
             channel.addCloseListener(listener);
 
-            TcpTransport.handleException(
-                channel,
-                exception,
-                lifecycle,
-                new OutboundHandler(
-                    randomAlphaOfLength(10),
-                    Version.CURRENT,
-                    new String[0],
-                    new StatsTracker(),
-                    testThreadPool,
-                    BigArrays.NON_RECYCLING_INSTANCE
-                )
-            );
+            TcpTransport.handleException(channel, exception, lifecycle, new OutboundHandler(new StatsTracker(), testThreadPool));
 
             if (expectClosed) {
                 assertTrue(listener.isDone());
diff --git a/server/src/test/java/org/opensearch/transport/TransportLoggerTests.java b/server/src/test/java/org/opensearch/transport/TransportLoggerTests.java
index 05296e9308657..cbd1b959f5f16 100644
--- a/server/src/test/java/org/opensearch/transport/TransportLoggerTests.java
+++ b/server/src/test/java/org/opensearch/transport/TransportLoggerTests.java
@@ -33,12 +33,6 @@
 
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
-import org.opensearch.Version;
-import org.opensearch.action.admin.cluster.stats.ClusterStatsAction;
-import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest;
-import org.opensearch.common.io.stream.BytesStreamOutput;
-import org.opensearch.common.settings.Settings;
-import org.opensearch.common.util.concurrent.ThreadContext;
 import org.opensearch.core.common.bytes.BytesReference;
 import org.opensearch.test.MockLogAppender;
 import org.opensearch.test.OpenSearchTestCase;
@@ -49,7 +43,7 @@
 import static org.mockito.Mockito.mock;
 
 @TestLogging(value = "org.opensearch.transport.TransportLogger:trace", reason = "to ensure we log network events on TRACE level")
-public class TransportLoggerTests extends OpenSearchTestCase {
+public abstract class TransportLoggerTests extends OpenSearchTestCase {
     public void testLoggingHandler() throws Exception {
         try (MockLogAppender appender = MockLogAppender.createForLoggers(LogManager.getLogger(TransportLogger.class))) {
             final String writePattern = ".*\\[length: \\d+"
@@ -90,20 +84,5 @@ public void testLoggingHandler() throws Exception {
         }
     }
 
-    private BytesReference buildRequest() throws IOException {
-        boolean compress = randomBoolean();
-        try (BytesStreamOutput bytesStreamOutput = new BytesStreamOutput()) {
-            OutboundMessage.Request request = new OutboundMessage.Request(
-                new ThreadContext(Settings.EMPTY),
-                new String[0],
-                new ClusterStatsRequest(),
-                Version.CURRENT,
-                ClusterStatsAction.NAME,
-                randomInt(30),
-                false,
-                compress
-            );
-            return request.serialize(bytesStreamOutput);
-        }
-    }
+    public abstract BytesReference buildRequest() throws IOException;
 }
diff --git a/server/src/test/java/org/opensearch/transport/CompressibleBytesOutputStreamTests.java b/server/src/test/java/org/opensearch/transport/nativeprotocol/CompressibleBytesOutputStreamTests.java
similarity index 99%
rename from server/src/test/java/org/opensearch/transport/CompressibleBytesOutputStreamTests.java
rename to server/src/test/java/org/opensearch/transport/nativeprotocol/CompressibleBytesOutputStreamTests.java
index 89018b7353e7c..eaa35469b9ec0 100644
--- a/server/src/test/java/org/opensearch/transport/CompressibleBytesOutputStreamTests.java
+++ b/server/src/test/java/org/opensearch/transport/nativeprotocol/CompressibleBytesOutputStreamTests.java
@@ -30,7 +30,7 @@
  * GitHub history for details.
  */
 
-package org.opensearch.transport;
+package org.opensearch.transport.nativeprotocol;
 
 import org.opensearch.common.io.stream.BytesStreamOutput;
 import org.opensearch.core.common.bytes.BytesReference;
diff --git a/server/src/test/java/org/opensearch/transport/nativeprotocol/NativeInboundDecoderTests.java b/server/src/test/java/org/opensearch/transport/nativeprotocol/NativeInboundDecoderTests.java
new file mode 100644
index 0000000000000..bd85939c753fa
--- /dev/null
+++ b/server/src/test/java/org/opensearch/transport/nativeprotocol/NativeInboundDecoderTests.java
@@ -0,0 +1,59 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.transport.nativeprotocol;
+
+import org.opensearch.Version;
+import org.opensearch.common.io.stream.BytesStreamOutput;
+import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.core.common.io.stream.Writeable;
+import org.opensearch.transport.InboundDecoderTests;
+
+import java.io.IOException;
+import java.util.Collections;
+
+public class NativeInboundDecoderTests extends InboundDecoderTests {
+
+    @Override
+    protected BytesReference serialize(
+        boolean isRequest,
+        Version version,
+        boolean handshake,
+        boolean compress,
+        String action,
+        long requestId,
+        Writeable transportMessage
+    ) throws IOException {
+        NativeOutboundMessage message;
+        if (isRequest) {
+            message = new NativeOutboundMessage.Request(
+                threadContext,
+                new String[0],
+                transportMessage,
+                version,
+                action,
+                requestId,
+                handshake,
+                compress
+            );
+        } else {
+            message = new NativeOutboundMessage.Response(
+                threadContext,
+                Collections.emptySet(),
+                transportMessage,
+                version,
+                requestId,
+                handshake,
+                compress
+            );
+        }
+
+        return message.serialize(new BytesStreamOutput());
+    }
+
+}
diff --git a/server/src/test/java/org/opensearch/transport/nativeprotocol/NativeInboundHandlerTests.java b/server/src/test/java/org/opensearch/transport/nativeprotocol/NativeInboundHandlerTests.java
new file mode 100644
index 0000000000000..ec0c1a50d5560
--- /dev/null
+++ b/server/src/test/java/org/opensearch/transport/nativeprotocol/NativeInboundHandlerTests.java
@@ -0,0 +1,45 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.transport.nativeprotocol;
+
+import org.opensearch.Version;
+import org.opensearch.common.io.stream.BytesStreamOutput;
+import org.opensearch.common.util.concurrent.ThreadContext;
+import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.core.common.io.stream.Writeable;
+import org.opensearch.transport.InboundHandlerTests;
+
+import java.io.IOException;
+
+public class NativeInboundHandlerTests extends InboundHandlerTests {
+
+    @Override
+    public BytesReference serializeOutboundRequest(
+        ThreadContext threadContext,
+        Writeable message,
+        Version version,
+        String action,
+        long requestId,
+        boolean compress,
+        boolean handshake
+    ) throws IOException {
+        NativeOutboundMessage.Request request = new NativeOutboundMessage.Request(
+            threadContext,
+            new String[0],
+            message,
+            version,
+            action,
+            requestId,
+            handshake,
+            compress
+        );
+        return request.serialize(new BytesStreamOutput());
+    }
+
+}
diff --git a/server/src/test/java/org/opensearch/transport/nativeprotocol/NativeInboundPipelineTests.java b/server/src/test/java/org/opensearch/transport/nativeprotocol/NativeInboundPipelineTests.java
new file mode 100644
index 0000000000000..6b5bf46eee3ae
--- /dev/null
+++ b/server/src/test/java/org/opensearch/transport/nativeprotocol/NativeInboundPipelineTests.java
@@ -0,0 +1,60 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.transport.nativeprotocol;
+
+import org.opensearch.Version;
+import org.opensearch.common.io.stream.BytesStreamOutput;
+import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.transport.InboundPipelineTests;
+import org.opensearch.transport.TestRequest;
+import org.opensearch.transport.TestResponse;
+
+import java.io.IOException;
+import java.util.Collections;
+
+public class NativeInboundPipelineTests extends InboundPipelineTests {
+
+    @Override
+    protected BytesReference serialize(
+        boolean isRequest,
+        Version version,
+        boolean handshake,
+        boolean compress,
+        String action,
+        long requestId,
+        String value
+    ) throws IOException {
+        NativeOutboundMessage message;
+        if (isRequest) {
+            message = new NativeOutboundMessage.Request(
+                threadContext,
+                new String[0],
+                new TestRequest(value),
+                version,
+                action,
+                requestId,
+                handshake,
+                compress
+            );
+        } else {
+            message = new NativeOutboundMessage.Response(
+                threadContext,
+                Collections.emptySet(),
+                new TestResponse(value),
+                version,
+                requestId,
+                handshake,
+                compress
+            );
+        }
+
+        return message.serialize(new BytesStreamOutput());
+    }
+
+}
diff --git a/server/src/test/java/org/opensearch/transport/nativeprotocol/NativeOutboundMessageTests.java b/server/src/test/java/org/opensearch/transport/nativeprotocol/NativeOutboundMessageTests.java
new file mode 100644
index 0000000000000..75c4e84b4456e
--- /dev/null
+++ b/server/src/test/java/org/opensearch/transport/nativeprotocol/NativeOutboundMessageTests.java
@@ -0,0 +1,50 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.transport.nativeprotocol;
+
+import org.opensearch.Version;
+import org.opensearch.common.io.stream.BytesStreamOutput;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.concurrent.ThreadContext;
+import org.opensearch.core.common.io.stream.BytesStreamInput;
+import org.opensearch.test.OpenSearchTestCase;
+import org.opensearch.transport.TestRequest;
+
+import java.io.IOException;
+
+public class NativeOutboundMessageTests extends OpenSearchTestCase {
+
+    public void testNativeOutboundMessageRequestSerialization() throws IOException {
+        NativeOutboundMessage.Request message = new NativeOutboundMessage.Request(
+            new ThreadContext(Settings.EMPTY),
+            new String[0],
+            new TestRequest("content"),
+            Version.CURRENT,
+            "action",
+            1,
+            false,
+            false
+        );
+        BytesStreamOutput output = new BytesStreamOutput();
+        message.serialize(output);
+
+        BytesStreamInput input = new BytesStreamInput(output.bytes().toBytesRef().bytes);
+        assertEquals(Version.CURRENT, input.getVersion());
+        // reading header details
+        assertEquals((byte) 'E', input.readByte());
+        assertEquals((byte) 'S', input.readByte());
+        assertNotEquals(0, input.readInt());
+        assertEquals(1, input.readLong());
+        assertEquals(0, input.readByte());
+        assertEquals(Version.CURRENT.id, input.readInt());
+        int variableHeaderSize = input.readInt();
+        assertNotEquals(-1, variableHeaderSize);
+    }
+
+}
diff --git a/server/src/test/java/org/opensearch/transport/nativeprotocol/NativeTransportLoggerTests.java b/server/src/test/java/org/opensearch/transport/nativeprotocol/NativeTransportLoggerTests.java
new file mode 100644
index 0000000000000..db75d6ff45556
--- /dev/null
+++ b/server/src/test/java/org/opensearch/transport/nativeprotocol/NativeTransportLoggerTests.java
@@ -0,0 +1,42 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.transport.nativeprotocol;
+
+import org.opensearch.Version;
+import org.opensearch.action.admin.cluster.stats.ClusterStatsAction;
+import org.opensearch.action.admin.cluster.stats.ClusterStatsRequest;
+import org.opensearch.common.io.stream.BytesStreamOutput;
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.util.concurrent.ThreadContext;
+import org.opensearch.core.common.bytes.BytesReference;
+import org.opensearch.test.junit.annotations.TestLogging;
+import org.opensearch.transport.TransportLoggerTests;
+
+import java.io.IOException;
+
+@TestLogging(value = "org.opensearch.transport.TransportLogger:trace", reason = "to ensure we log network events on TRACE level")
+public class NativeTransportLoggerTests extends TransportLoggerTests {
+
+    public BytesReference buildRequest() throws IOException {
+        boolean compress = randomBoolean();
+        try (BytesStreamOutput bytesStreamOutput = new BytesStreamOutput()) {
+            NativeOutboundMessage.Request request = new NativeOutboundMessage.Request(
+                new ThreadContext(Settings.EMPTY),
+                new String[0],
+                new ClusterStatsRequest(),
+                Version.CURRENT,
+                ClusterStatsAction.NAME,
+                randomInt(30),
+                false,
+                compress
+            );
+            return request.serialize(new BytesStreamOutput());
+        }
+    }
+}
diff --git a/settings.gradle b/settings.gradle
index 8fbf32504215b..065a8dc3d0a8a 100644
--- a/settings.gradle
+++ b/settings.gradle
@@ -10,7 +10,7 @@
  */
 
 plugins {
-  id "com.gradle.enterprise" version "3.16.2"
+  id "com.gradle.enterprise" version "3.17.3"
 }
 
 ext.disableBuildCache = hasProperty('DISABLE_BUILD_CACHE') || System.getenv().containsKey('DISABLE_BUILD_CACHE')
diff --git a/test/fixtures/hdfs-fixture/build.gradle b/test/fixtures/hdfs-fixture/build.gradle
index 0ca4797bfeff1..33635f2053a21 100644
--- a/test/fixtures/hdfs-fixture/build.gradle
+++ b/test/fixtures/hdfs-fixture/build.gradle
@@ -37,7 +37,7 @@ versions << [
 ]
 
 dependencies {
-  api("org.apache.hadoop:hadoop-minicluster:3.3.6") {
+  api("org.apache.hadoop:hadoop-minicluster:3.4.0") {
     exclude module: 'websocket-client'
     exclude module: 'jettison'
     exclude module: 'netty'
@@ -62,14 +62,14 @@ dependencies {
   api "com.fasterxml.jackson.jaxrs:jackson-jaxrs-json-provider:${versions.jackson}"
   api "com.fasterxml.jackson.core:jackson-databind:${versions.jackson_databind}"
   api "com.fasterxml.woodstox:woodstox-core:${versions.woodstox}"
-  api 'net.minidev:json-smart:2.5.0'
+  api 'net.minidev:json-smart:2.5.1'
   api "org.mockito:mockito-core:${versions.mockito}"
   api "com.google.protobuf:protobuf-java:${versions.protobuf}"
   api "org.jetbrains.kotlin:kotlin-stdlib:${versions.kotlin}"
   api "org.eclipse.jetty:jetty-server:${versions.jetty}"
   api "org.eclipse.jetty.websocket:javax-websocket-server-impl:${versions.jetty}"
   api 'org.apache.zookeeper:zookeeper:3.9.2'
-  api "org.apache.commons:commons-text:1.11.0"
+  api "org.apache.commons:commons-text:1.12.0"
   api "commons-net:commons-net:3.10.0"
   api "ch.qos.logback:logback-core:1.5.3"
   api "ch.qos.logback:logback-classic:1.2.13"
@@ -78,6 +78,6 @@ dependencies {
   runtimeOnly("com.squareup.okhttp3:okhttp:4.12.0") {
     exclude group: "com.squareup.okio"
   }
-  runtimeOnly "com.squareup.okio:okio:3.8.0"
+  runtimeOnly "com.squareup.okio:okio:3.9.0"
   runtimeOnly "org.xerial.snappy:snappy-java:1.1.10.5"
 }
diff --git a/test/framework/src/main/java/org/opensearch/cluster/MockInternalClusterInfoService.java b/test/framework/src/main/java/org/opensearch/cluster/MockInternalClusterInfoService.java
index 1ad6083074025..35ca5d80aeb4e 100644
--- a/test/framework/src/main/java/org/opensearch/cluster/MockInternalClusterInfoService.java
+++ b/test/framework/src/main/java/org/opensearch/cluster/MockInternalClusterInfoService.java
@@ -124,7 +124,8 @@ List<NodeStats> adjustNodesStats(List<NodeStats> nodesStats) {
                 nodeStats.getSearchPipelineStats(),
                 nodeStats.getSegmentReplicationRejectionStats(),
                 nodeStats.getRepositoriesStats(),
-                nodeStats.getAdmissionControlStats()
+                nodeStats.getAdmissionControlStats(),
+                nodeStats.getNodeCacheStats()
             );
         }).collect(Collectors.toList());
     }
diff --git a/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java b/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java
index afebd0b70bf06..3ae737bf63923 100644
--- a/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java
@@ -39,6 +39,7 @@
 import org.apache.logging.log4j.message.ParameterizedMessage;
 import org.opensearch.OpenSearchException;
 import org.opensearch.Version;
+import org.opensearch.cluster.ClusterManagerMetrics;
 import org.opensearch.cluster.ClusterModule;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.ClusterStateTaskListener;
@@ -89,7 +90,6 @@
 import org.opensearch.monitor.StatusInfo;
 import org.opensearch.node.remotestore.RemoteStoreNodeService;
 import org.opensearch.repositories.RepositoriesService;
-import org.opensearch.telemetry.metrics.MetricsRegistry;
 import org.opensearch.telemetry.metrics.noop.NoopMetricsRegistry;
 import org.opensearch.telemetry.tracing.noop.NoopTracer;
 import org.opensearch.test.OpenSearchTestCase;
@@ -1147,7 +1147,7 @@ protected Optional<DisruptableMockTransport> getDisruptableMockTransport(Transpo
                     clusterSettings,
                     deterministicTaskQueue,
                     threadPool,
-                    NoopMetricsRegistry.INSTANCE
+                    new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE)
                 );
                 clusterService = new ClusterService(settings, clusterSettings, clusterManagerService, clusterApplierService);
                 clusterService.setNodeConnectionsService(
@@ -1598,9 +1598,9 @@ static class DisruptableClusterApplierService extends ClusterApplierService {
             ClusterSettings clusterSettings,
             DeterministicTaskQueue deterministicTaskQueue,
             ThreadPool threadPool,
-            MetricsRegistry metricsRegistry
+            ClusterManagerMetrics clusterManagerMetrics
         ) {
-            super(nodeName, settings, clusterSettings, threadPool, metricsRegistry);
+            super(nodeName, settings, clusterSettings, threadPool, clusterManagerMetrics);
             this.nodeName = nodeName;
             this.deterministicTaskQueue = deterministicTaskQueue;
             addStateApplier(event -> {
diff --git a/test/framework/src/main/java/org/opensearch/cluster/service/FakeThreadPoolClusterManagerService.java b/test/framework/src/main/java/org/opensearch/cluster/service/FakeThreadPoolClusterManagerService.java
index 337ecde98bf6e..53ef595c7931e 100644
--- a/test/framework/src/main/java/org/opensearch/cluster/service/FakeThreadPoolClusterManagerService.java
+++ b/test/framework/src/main/java/org/opensearch/cluster/service/FakeThreadPoolClusterManagerService.java
@@ -34,6 +34,7 @@
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.opensearch.cluster.ClusterChangedEvent;
+import org.opensearch.cluster.ClusterManagerMetrics;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.coordination.ClusterStatePublisher.AckListener;
 import org.opensearch.common.UUIDs;
@@ -76,7 +77,7 @@ public FakeThreadPoolClusterManagerService(
             Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), nodeName).build(),
             new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
             threadPool,
-            NoopMetricsRegistry.INSTANCE
+            new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE)
         );
         this.name = serviceName;
         this.onTaskAvailableToRun = onTaskAvailableToRun;
diff --git a/test/framework/src/main/java/org/opensearch/index/engine/EngineTestCase.java b/test/framework/src/main/java/org/opensearch/index/engine/EngineTestCase.java
index 43289a7c89524..3403425d89254 100644
--- a/test/framework/src/main/java/org/opensearch/index/engine/EngineTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/index/engine/EngineTestCase.java
@@ -527,7 +527,14 @@ protected Translog createTranslog(LongSupplier primaryTermSupplier) throws IOExc
     }
 
     protected Translog createTranslog(Path translogPath, LongSupplier primaryTermSupplier) throws IOException {
-        TranslogConfig translogConfig = new TranslogConfig(shardId, translogPath, INDEX_SETTINGS, BigArrays.NON_RECYCLING_INSTANCE, "");
+        TranslogConfig translogConfig = new TranslogConfig(
+            shardId,
+            translogPath,
+            INDEX_SETTINGS,
+            BigArrays.NON_RECYCLING_INSTANCE,
+            "",
+            false
+        );
         String translogUUID = Translog.createEmptyTranslog(
             translogPath,
             SequenceNumbers.NO_OPS_PERFORMED,
@@ -877,7 +884,8 @@ public EngineConfig config(
             translogPath,
             indexSettings,
             BigArrays.NON_RECYCLING_INSTANCE,
-            ""
+            "",
+            false
         );
         final List<ReferenceManager.RefreshListener> extRefreshListenerList = externalRefreshListener == null
             ? emptyList()
@@ -898,7 +906,8 @@ public EngineConfig config(
                 update -> {},
                 () -> 0L,
                 (leases, listener) -> listener.onResponse(new ReplicationResponse()),
-                () -> SafeCommitInfo.EMPTY
+                () -> SafeCommitInfo.EMPTY,
+                sId -> false
             );
             globalCheckpointSupplier = replicationTracker;
             retentionLeasesSupplier = replicationTracker::getRetentionLeases;
@@ -945,7 +954,14 @@ protected EngineConfig config(
                 .put(IndexSettings.INDEX_SOFT_DELETES_SETTING.getKey(), true)
                 .build()
         );
-        TranslogConfig translogConfig = new TranslogConfig(shardId, translogPath, indexSettings, BigArrays.NON_RECYCLING_INSTANCE, "");
+        TranslogConfig translogConfig = new TranslogConfig(
+            shardId,
+            translogPath,
+            indexSettings,
+            BigArrays.NON_RECYCLING_INSTANCE,
+            "",
+            false
+        );
         return new EngineConfig.Builder().shardId(config.getShardId())
             .threadPool(config.getThreadPool())
             .indexSettings(indexSettings)
diff --git a/test/framework/src/main/java/org/opensearch/index/mapper/MapperServiceTestCase.java b/test/framework/src/main/java/org/opensearch/index/mapper/MapperServiceTestCase.java
index a65ce3cbdd380..bd16e4f65e159 100644
--- a/test/framework/src/main/java/org/opensearch/index/mapper/MapperServiceTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/index/mapper/MapperServiceTestCase.java
@@ -225,13 +225,32 @@ protected final XContentBuilder mapping(CheckedConsumer<XContentBuilder, IOExcep
         return builder.endObject().endObject().endObject();
     }
 
+    protected final XContentBuilder derivedMapping(CheckedConsumer<XContentBuilder, IOException> buildFields) throws IOException {
+        XContentBuilder builder = XContentFactory.jsonBuilder().startObject().startObject("_doc").startObject("derived");
+        buildFields.accept(builder);
+        return builder.endObject().endObject().endObject();
+    }
+
     protected final XContentBuilder dynamicMapping(Mapping dynamicMapping) throws IOException {
         XContentBuilder builder = XContentFactory.jsonBuilder().startObject();
         dynamicMapping.toXContent(builder, ToXContent.EMPTY_PARAMS);
         return builder.endObject();
     }
 
-    protected final XContentBuilder fieldMapping(CheckedConsumer<XContentBuilder, IOException> buildField) throws IOException {
+    protected XContentBuilder fieldMapping(CheckedConsumer<XContentBuilder, IOException> buildField) throws IOException {
+        return fieldMapping(buildField, false);
+    }
+
+    protected final XContentBuilder fieldMapping(CheckedConsumer<XContentBuilder, IOException> buildField, Boolean isDerived)
+        throws IOException {
+        if (isDerived) {
+            return derivedMapping(b -> {
+                b.startObject("field");
+                buildField.accept(b);
+                b.endObject();
+            });
+        }
+
         return mapping(b -> {
             b.startObject("field");
             buildField.accept(b);
@@ -247,6 +266,9 @@ protected QueryShardContext createQueryShardContext(MapperService mapperService)
         when(queryShardContext.getSearchQuoteAnalyzer(any())).thenCallRealMethod();
         when(queryShardContext.getSearchAnalyzer(any())).thenCallRealMethod();
         when(queryShardContext.getIndexSettings()).thenReturn(mapperService.getIndexSettings());
+        when(queryShardContext.getObjectMapper(anyString())).thenAnswer(
+            inv -> mapperService.getObjectMapper(inv.getArguments()[0].toString())
+        );
         when(queryShardContext.simpleMatchToIndexNames(any())).thenAnswer(
             inv -> mapperService.simpleMatchToFullName(inv.getArguments()[0].toString())
         );
diff --git a/test/framework/src/main/java/org/opensearch/index/replication/OpenSearchIndexLevelReplicationTestCase.java b/test/framework/src/main/java/org/opensearch/index/replication/OpenSearchIndexLevelReplicationTestCase.java
index e6e20ce8f8566..a5dc13c334513 100644
--- a/test/framework/src/main/java/org/opensearch/index/replication/OpenSearchIndexLevelReplicationTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/index/replication/OpenSearchIndexLevelReplicationTestCase.java
@@ -64,6 +64,7 @@
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.cluster.node.DiscoveryNodeRole;
+import org.opensearch.cluster.node.DiscoveryNodes;
 import org.opensearch.cluster.routing.AllocationId;
 import org.opensearch.cluster.routing.IndexShardRoutingTable;
 import org.opensearch.cluster.routing.RecoverySource;
@@ -96,6 +97,7 @@
 import org.opensearch.index.seqno.RetentionLeases;
 import org.opensearch.index.shard.IndexShard;
 import org.opensearch.index.shard.IndexShardTestCase;
+import org.opensearch.index.shard.IndexShardTestUtils;
 import org.opensearch.index.shard.PrimaryReplicaSyncer;
 import org.opensearch.index.shard.ShardPath;
 import org.opensearch.index.translog.Translog;
@@ -340,6 +342,23 @@ public synchronized void startAll() throws IOException {
             startReplicas(replicas.size());
         }
 
+        public synchronized DiscoveryNodes generateFakeDiscoveryNodes() {
+            DiscoveryNodes.Builder builder = new DiscoveryNodes.Builder();
+            if (primary.indexSettings() != null && primary.indexSettings().isAssignedOnRemoteNode()) {
+                builder.add(IndexShardTestUtils.getFakeRemoteEnabledNode(primary.routingEntry().currentNodeId()));
+            } else {
+                builder.add(IndexShardTestUtils.getFakeDiscoNode(primary.routingEntry().currentNodeId()));
+            }
+            for (IndexShard replica : replicas) {
+                if (replica.indexSettings() != null && replica.indexSettings().isAssignedOnRemoteNode()) {
+                    builder.add(IndexShardTestUtils.getFakeRemoteEnabledNode(replica.routingEntry().currentNodeId()));
+                } else {
+                    builder.add(IndexShardTestUtils.getFakeDiscoNode(replica.routingEntry().currentNodeId()));
+                }
+            }
+            return builder.build();
+        }
+
         public synchronized int startReplicas(int numOfReplicasToStart) throws IOException {
             if (primary.routingEntry().initializing()) {
                 startPrimary();
@@ -371,7 +390,8 @@ public void startPrimary() throws IOException {
                 null,
                 currentClusterStateVersion.incrementAndGet(),
                 activeIds,
-                routingTable
+                routingTable,
+                generateFakeDiscoveryNodes()
             );
             for (final IndexShard replica : replicas) {
                 recoverReplica(replica);
@@ -492,7 +512,8 @@ public synchronized void promoteReplicaToPrimary(
                 primaryReplicaSyncer,
                 currentClusterStateVersion.incrementAndGet(),
                 activeIds(),
-                routingTable
+                routingTable,
+                generateFakeDiscoveryNodes()
             );
         }
 
@@ -638,14 +659,16 @@ public void syncGlobalCheckpoint() {
         }
 
         private void updateAllocationIDsOnPrimary() throws IOException {
-
             primary.updateShardState(
                 primary.routingEntry(),
                 primary.getPendingPrimaryTerm(),
                 null,
                 currentClusterStateVersion.incrementAndGet(),
                 activeIds(),
-                routingTable(Function.identity())
+                routingTable(Function.identity()),
+                primary.indexSettings().isRemoteTranslogStoreEnabled()
+                    ? IndexShardTestUtils.getFakeRemoteEnabledDiscoveryNodes(routingTable(Function.identity()).getShards())
+                    : IndexShardTestUtils.getFakeDiscoveryNodes(routingTable(Function.identity()).getShards())
             );
         }
 
diff --git a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java
index a2f9eb677c0ac..6b609d8af62a1 100644
--- a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestCase.java
@@ -48,6 +48,7 @@
 import org.opensearch.cluster.metadata.RepositoryMetadata;
 import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.cluster.node.DiscoveryNodeRole;
+import org.opensearch.cluster.node.DiscoveryNodes;
 import org.opensearch.cluster.routing.IndexShardRoutingTable;
 import org.opensearch.cluster.routing.RecoverySource;
 import org.opensearch.cluster.routing.ShardRouting;
@@ -116,6 +117,7 @@
 import org.opensearch.index.translog.RemoteBlobStoreInternalTranslogFactory;
 import org.opensearch.index.translog.Translog;
 import org.opensearch.index.translog.TranslogFactory;
+import org.opensearch.indices.DefaultRemoteStoreSettings;
 import org.opensearch.indices.IndicesService;
 import org.opensearch.indices.breaker.HierarchyCircuitBreakerService;
 import org.opensearch.indices.recovery.AsyncRecoveryTarget;
@@ -387,7 +389,7 @@ protected IndexShard newShard(ShardId shardId, boolean primary, IndexingOperatio
     }
 
     /**
-     * creates a new initializing shard. The shard will will be put in its proper path under the
+     * creates a new initializing shard. The shard will be put in its proper path under the
      * supplied node id.
      *
      * @param shardId the shard id to use
@@ -405,7 +407,7 @@ protected IndexShard newShard(
     }
 
     /**
-     * creates a new initializing shard. The shard will will be put in its proper path under the
+     * creates a new initializing shard. The shard will be put in its proper path under the
      * supplied node id.
      *
      * @param shardId the shard id to use
@@ -439,7 +441,7 @@ protected IndexShard newShard(
     }
 
     /**
-     * creates a new initializing shard. The shard will will be put in its proper path under the
+     * creates a new initializing shard. The shard will be put in its proper path under the
      * current node id the shard is assigned to.
      *
      * @param routing       shard routing to use
@@ -457,7 +459,7 @@ protected IndexShard newShard(
     }
 
     /**
-     * creates a new initializing shard. The shard will will be put in its proper path under the
+     * creates a new initializing shard. The shard will be put in its proper path under the
      * current node id the shard is assigned to.
      * @param routing                shard routing to use
      * @param indexMetadata          indexMetadata for the shard, including any mapping
@@ -496,7 +498,7 @@ protected IndexShard newShard(
     }
 
     /**
-     * creates a new initializing shard. The shard will will be put in its proper path under the
+     * creates a new initializing shard. The shard will be put in its proper path under the
      * current node id the shard is assigned to.
      * @param routing                       shard routing to use
      * @param shardPath                     path to use for shard data
@@ -618,12 +620,14 @@ protected IndexShard newShard(
         IndexingOperationListener... listeners
     ) throws IOException {
         Settings nodeSettings = Settings.builder().put("node.name", routing.currentNodeId()).build();
+        DiscoveryNodes discoveryNodes = IndexShardTestUtils.getFakeDiscoveryNodes(routing);
         // To simulate that the node is remote backed
         if (indexMetadata.getSettings().get(IndexMetadata.SETTING_REMOTE_STORE_ENABLED) == "true") {
             nodeSettings = Settings.builder()
                 .put("node.name", routing.currentNodeId())
                 .put("node.attr.remote_store.translog.repository", "seg_repo")
                 .build();
+            discoveryNodes = DiscoveryNodes.builder().add(IndexShardTestUtils.getFakeRemoteEnabledNode(routing.currentNodeId())).build();
         }
         final IndexSettings indexSettings = new IndexSettings(indexMetadata, nodeSettings);
         final IndexShard indexShard;
@@ -653,7 +657,7 @@ protected IndexShard newShard(
             RemoteStoreStatsTrackerFactory remoteStoreStatsTrackerFactory = null;
             RepositoriesService mockRepoSvc = mock(RepositoriesService.class);
 
-            if (indexSettings.isRemoteStoreEnabled() || indexSettings.isRemoteNode()) {
+            if (indexSettings.isRemoteStoreEnabled() || indexSettings.isAssignedOnRemoteNode()) {
                 String remoteStoreRepository = indexSettings.getRemoteStoreRepository();
                 // remote path via setting a repository . This is a hack used for shards are created using reset .
                 // since we can't get remote path from IndexShard directly, we are using repository to store it .
@@ -678,7 +682,8 @@ protected IndexShard newShard(
                         () -> mockRepoSvc,
                         threadPool,
                         settings.getRemoteStoreTranslogRepository(),
-                        new RemoteTranslogTransferTracker(shardRouting.shardId(), 20)
+                        new RemoteTranslogTransferTracker(shardRouting.shardId(), 20),
+                        DefaultRemoteStoreSettings.INSTANCE
                     );
                 }
                 return new InternalTranslogFactory();
@@ -708,10 +713,11 @@ protected IndexShard newShard(
                 checkpointPublisher,
                 remoteStore,
                 remoteStoreStatsTrackerFactory,
-                () -> IndexSettings.DEFAULT_REMOTE_TRANSLOG_BUFFER_INTERVAL,
                 "dummy-node",
                 DefaultRecoverySettings.INSTANCE,
-                false
+                DefaultRemoteStoreSettings.INSTANCE,
+                false,
+                discoveryNodes
             );
             indexShard.addShardFailureCallback(DEFAULT_SHARD_FAILURE_HANDLER);
             if (remoteStoreStatsTrackerFactory != null) {
@@ -985,7 +991,7 @@ protected void closeShards(Iterable<IndexShard> shards) throws IOException {
     protected void recoverShardFromStore(IndexShard primary) throws IOException {
         primary.markAsRecovering(
             "store",
-            new RecoveryState(primary.routingEntry(), getFakeDiscoNode(primary.routingEntry().currentNodeId()), null)
+            new RecoveryState(primary.routingEntry(), IndexShardTestUtils.getFakeDiscoNode(primary.routingEntry().currentNodeId()), null)
         );
         recoverFromStore(primary);
         updateRoutingEntry(primary, ShardRoutingHelper.moveToStarted(primary.routingEntry()));
@@ -1002,7 +1008,19 @@ public static void updateRoutingEntry(IndexShard shard, ShardRouting shardRoutin
             null,
             currentClusterStateVersion.incrementAndGet(),
             inSyncIds,
-            newRoutingTable
+            newRoutingTable,
+            DiscoveryNodes.builder()
+                .add(
+                    new DiscoveryNode(
+                        shardRouting.currentNodeId(),
+                        shardRouting.currentNodeId(),
+                        buildNewFakeTransportAddress(),
+                        Collections.emptyMap(),
+                        DiscoveryNodeRole.BUILT_IN_ROLES,
+                        Version.CURRENT
+                    )
+                )
+                .build()
         );
     }
 
@@ -1016,17 +1034,6 @@ protected void recoveryEmptyReplica(IndexShard replica, boolean startReplica) th
         }
     }
 
-    protected DiscoveryNode getFakeDiscoNode(String id) {
-        return new DiscoveryNode(
-            id,
-            id,
-            buildNewFakeTransportAddress(),
-            Collections.emptyMap(),
-            DiscoveryNodeRole.BUILT_IN_ROLES,
-            Version.CURRENT
-        );
-    }
-
     protected void recoverReplica(IndexShard replica, IndexShard primary, boolean startReplica) throws IOException {
         recoverReplica(replica, primary, startReplica, getReplicationFunc(replica));
     }
@@ -1103,7 +1110,7 @@ protected void recoverReplica(
      * @param targetSupplier         supplies an instance of {@link RecoveryTarget}
      * @param markAsRecovering       set to {@code false} if the replica is marked as recovering
      */
-    protected final void recoverUnstartedReplica(
+    public final void recoverUnstartedReplica(
         final IndexShard replica,
         final IndexShard primary,
         final BiFunction<IndexShard, DiscoveryNode, RecoveryTarget> targetSupplier,
@@ -1112,8 +1119,18 @@ protected final void recoverUnstartedReplica(
         final IndexShardRoutingTable routingTable,
         final Function<List<IndexShard>, List<SegmentReplicationTarget>> replicatePrimaryFunction
     ) throws IOException {
-        final DiscoveryNode pNode = getFakeDiscoNode(primary.routingEntry().currentNodeId());
-        final DiscoveryNode rNode = getFakeDiscoNode(replica.routingEntry().currentNodeId());
+        final DiscoveryNode pNode;
+        final DiscoveryNode rNode;
+        if (primary.isRemoteTranslogEnabled()) {
+            pNode = IndexShardTestUtils.getFakeRemoteEnabledNode(primary.routingEntry().currentNodeId());
+        } else {
+            pNode = IndexShardTestUtils.getFakeDiscoNode(primary.routingEntry().currentNodeId());
+        }
+        if (replica.isRemoteTranslogEnabled()) {
+            rNode = IndexShardTestUtils.getFakeRemoteEnabledNode(replica.routingEntry().currentNodeId());
+        } else {
+            rNode = IndexShardTestUtils.getFakeDiscoNode(replica.routingEntry().currentNodeId());
+        }
         if (markAsRecovering) {
             replica.markAsRecovering("remote", new RecoveryState(replica.routingEntry(), pNode, rNode));
         } else {
@@ -1154,7 +1171,10 @@ protected final void recoverUnstartedReplica(
             null,
             currentClusterStateVersion.incrementAndGet(),
             inSyncIds,
-            routingTable
+            routingTable,
+            primary.isRemoteTranslogEnabled()
+                ? IndexShardTestUtils.getFakeRemoteEnabledDiscoveryNodes(routingTable.getShards())
+                : IndexShardTestUtils.getFakeDiscoveryNodes(routingTable.getShards())
         );
         try {
             PlainActionFuture<RecoveryResponse> future = new PlainActionFuture<>();
@@ -1188,7 +1208,10 @@ protected void startReplicaAfterRecovery(
             null,
             currentClusterStateVersion.incrementAndGet(),
             inSyncIdsWithReplica,
-            newRoutingTable
+            newRoutingTable,
+            primary.indexSettings.isRemoteTranslogStoreEnabled()
+                ? IndexShardTestUtils.getFakeRemoteEnabledDiscoveryNodes(routingTable.shards())
+                : IndexShardTestUtils.getFakeDiscoveryNodes(routingTable.shards())
         );
         replica.updateShardState(
             replica.routingEntry().moveToStarted(),
@@ -1196,7 +1219,10 @@ protected void startReplicaAfterRecovery(
             null,
             currentClusterStateVersion.get(),
             inSyncIdsWithReplica,
-            newRoutingTable
+            newRoutingTable,
+            replica.indexSettings.isRemoteTranslogStoreEnabled()
+                ? IndexShardTestUtils.getFakeRemoteEnabledDiscoveryNodes(routingTable.shards())
+                : IndexShardTestUtils.getFakeDiscoveryNodes(routingTable.shards())
         );
     }
 
@@ -1225,7 +1251,8 @@ protected void promoteReplica(IndexShard replica, Set<String> inSyncIds, IndexSh
             ),
             currentClusterStateVersion.incrementAndGet(),
             inSyncIds,
-            newRoutingTable
+            newRoutingTable,
+            IndexShardTestUtils.getFakeDiscoveryNodes(routingEntry)
         );
     }
 
@@ -1369,7 +1396,7 @@ protected void recoverShardFromSnapshot(final IndexShard shard, final Snapshot s
         final Version version = Version.CURRENT;
         final ShardId shardId = shard.shardId();
         final IndexId indexId = new IndexId(shardId.getIndex().getName(), shardId.getIndex().getUUID());
-        final DiscoveryNode node = getFakeDiscoNode(shard.routingEntry().currentNodeId());
+        final DiscoveryNode node = IndexShardTestUtils.getFakeDiscoNode(shard.routingEntry().currentNodeId());
         final RecoverySource.SnapshotRecoverySource recoverySource = new RecoverySource.SnapshotRecoverySource(
             UUIDs.randomBase64UUID(),
             snapshot,
@@ -1497,7 +1524,7 @@ private SegmentReplicationTargetService prepareForReplication(
 
         SegmentReplicationSourceFactory sourceFactory = null;
         SegmentReplicationTargetService targetService;
-        if (primaryShard.indexSettings.isRemoteStoreEnabled() || primaryShard.indexSettings.isRemoteNode()) {
+        if (primaryShard.indexSettings.isRemoteStoreEnabled() || primaryShard.indexSettings.isAssignedOnRemoteNode()) {
             RecoverySettings recoverySettings = new RecoverySettings(
                 Settings.EMPTY,
                 new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS)
@@ -1611,12 +1638,10 @@ public void getCheckpointMetadata(
                 ReplicationCheckpoint checkpoint,
                 ActionListener<CheckpointInfoResponse> listener
             ) {
-                try {
-                    final CopyState copyState = new CopyState(primaryShard.getLatestReplicationCheckpoint(), primaryShard);
+                try (final CopyState copyState = new CopyState(primaryShard)) {
                     listener.onResponse(
                         new CheckpointInfoResponse(copyState.getCheckpoint(), copyState.getMetadataMap(), copyState.getInfosBytes())
                     );
-                    copyState.decRef();
                 } catch (IOException e) {
                     logger.error("Unexpected error computing CopyState", e);
                     Assert.fail("Failed to compute copyState");
diff --git a/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestUtils.java b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestUtils.java
new file mode 100644
index 0000000000000..abf8f2a4da6c1
--- /dev/null
+++ b/test/framework/src/main/java/org/opensearch/index/shard/IndexShardTestUtils.java
@@ -0,0 +1,71 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.index.shard;
+
+import org.opensearch.Version;
+import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.cluster.node.DiscoveryNodeRole;
+import org.opensearch.cluster.node.DiscoveryNodes;
+import org.opensearch.cluster.routing.ShardRouting;
+import org.opensearch.node.remotestore.RemoteStoreNodeAttribute;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class IndexShardTestUtils {
+    public static final String MOCK_SEGMENT_REPO_NAME = "segment-test-repo";
+    public static final String MOCK_TLOG_REPO_NAME = "tlog-test-repo";
+
+    public static DiscoveryNode getFakeDiscoNode(String id) {
+        return new DiscoveryNode(
+            id,
+            id,
+            IndexShardTestCase.buildNewFakeTransportAddress(),
+            Collections.emptyMap(),
+            DiscoveryNodeRole.BUILT_IN_ROLES,
+            Version.CURRENT
+        );
+    }
+
+    public static DiscoveryNode getFakeRemoteEnabledNode(String id) {
+        Map<String, String> remoteNodeAttributes = new HashMap<String, String>();
+        remoteNodeAttributes.put(RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY, MOCK_SEGMENT_REPO_NAME);
+        remoteNodeAttributes.put(RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY, MOCK_TLOG_REPO_NAME);
+        return new DiscoveryNode(
+            id,
+            id,
+            IndexShardTestCase.buildNewFakeTransportAddress(),
+            remoteNodeAttributes,
+            DiscoveryNodeRole.BUILT_IN_ROLES,
+            Version.CURRENT
+        );
+    }
+
+    public static DiscoveryNodes getFakeDiscoveryNodes(List<ShardRouting> shardRoutings) {
+        DiscoveryNodes.Builder builder = DiscoveryNodes.builder();
+        for (ShardRouting routing : shardRoutings) {
+            builder.add(getFakeDiscoNode(routing.currentNodeId()));
+        }
+        return builder.build();
+    }
+
+    public static DiscoveryNodes getFakeRemoteEnabledDiscoveryNodes(List<ShardRouting> shardRoutings) {
+        DiscoveryNodes.Builder builder = DiscoveryNodes.builder();
+        for (ShardRouting routing : shardRoutings) {
+            builder.add(getFakeRemoteEnabledNode(routing.currentNodeId()));
+        }
+        return builder.build();
+    }
+
+    public static DiscoveryNodes getFakeDiscoveryNodes(ShardRouting shardRouting) {
+        return DiscoveryNodes.builder().add(getFakeDiscoNode(shardRouting.currentNodeId())).build();
+    }
+}
diff --git a/test/framework/src/main/java/org/opensearch/indices/analysis/AnalysisFactoryTestCase.java b/test/framework/src/main/java/org/opensearch/indices/analysis/AnalysisFactoryTestCase.java
index c412ae8317f24..5231fe095f0f0 100644
--- a/test/framework/src/main/java/org/opensearch/indices/analysis/AnalysisFactoryTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/indices/analysis/AnalysisFactoryTestCase.java
@@ -222,6 +222,9 @@ public abstract class AnalysisFactoryTestCase extends OpenSearchTestCase {
         .put("persianstem", Void.class)
         // https://github.com/apache/lucene/pull/12169
         .put("word2vecsynonym", Void.class)
+        // https://github.com/apache/lucene/pull/12915
+        .put("japanesehiraganauppercase", Void.class)
+        .put("japanesekatakanauppercase", Void.class)
         .immutableMap();
 
     static final Map<String, Class<?>> KNOWN_CHARFILTERS = new MapBuilder<String, Class<?>>()
diff --git a/test/framework/src/main/java/org/opensearch/ingest/RandomDocumentPicks.java b/test/framework/src/main/java/org/opensearch/ingest/RandomDocumentPicks.java
index c478bf9239f74..0e42787b16be8 100644
--- a/test/framework/src/main/java/org/opensearch/ingest/RandomDocumentPicks.java
+++ b/test/framework/src/main/java/org/opensearch/ingest/RandomDocumentPicks.java
@@ -71,6 +71,17 @@ public static String randomFieldName(Random random) {
         return fieldName.toString();
     }
 
+    /**
+     * Returns a random field name that doesn't exist in the document.
+     */
+    public static String randomNonExistingFieldName(Random random, IngestDocument ingestDocument) {
+        String fieldName;
+        do {
+            fieldName = randomFieldName(random);
+        } while (canAddField(fieldName, ingestDocument) == false);
+        return fieldName;
+    }
+
     /**
      * Returns a random leaf field name.
      */
diff --git a/test/framework/src/main/java/org/opensearch/script/MockScriptEngine.java b/test/framework/src/main/java/org/opensearch/script/MockScriptEngine.java
index 83b245a1bcecb..048f0acb60cde 100644
--- a/test/framework/src/main/java/org/opensearch/script/MockScriptEngine.java
+++ b/test/framework/src/main/java/org/opensearch/script/MockScriptEngine.java
@@ -35,6 +35,7 @@
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.IndexSearcher;
 import org.apache.lucene.search.Scorable;
+import org.opensearch.common.collect.Tuple;
 import org.opensearch.index.query.IntervalFilterScript;
 import org.opensearch.index.similarity.ScriptedSimilarity.Doc;
 import org.opensearch.index.similarity.ScriptedSimilarity.Field;
@@ -43,8 +44,10 @@
 import org.opensearch.search.aggregations.pipeline.MovingFunctionScript;
 import org.opensearch.search.lookup.LeafSearchLookup;
 import org.opensearch.search.lookup.SearchLookup;
+import org.opensearch.search.lookup.SourceLookup;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
@@ -281,7 +284,45 @@ public double execute(Map<String, Object> params1, double[] values) {
         } else if (context.instanceClazz.equals(IntervalFilterScript.class)) {
             IntervalFilterScript.Factory factory = mockCompiled::createIntervalFilterScript;
             return context.factoryClazz.cast(factory);
+        } else if (context.instanceClazz.equals(DerivedFieldScript.class)) {
+            DerivedFieldScript.Factory factory = new DerivedFieldScript.Factory() {
+                @Override
+                public boolean isResultDeterministic() {
+                    return true;
+                }
+
+                @Override
+                public DerivedFieldScript.LeafFactory newFactory(Map<String, Object> derivedFieldParams, SearchLookup lookup) {
+                    return ctx -> new DerivedFieldScript(derivedFieldParams, lookup, ctx) {
+                        @Override
+                        public void execute() {
+                            Map<String, Object> vars = new HashMap<>(derivedFieldParams);
+                            SourceLookup sourceLookup = lookup.source();
+                            vars.put("params", derivedFieldParams);
+                            vars.put("_source", sourceLookup.loadSourceIfNeeded());
+                            Object result = script.apply(vars);
+                            if (result instanceof ArrayList) {
+                                for (Object v : ((ArrayList<?>) result)) {
+                                    if (v instanceof HashMap) {
+                                        addEmittedValue(new Tuple(((HashMap<?, ?>) v).get("lat"), ((HashMap<?, ?>) v).get("lon")));
+                                    } else {
+                                        addEmittedValue(v);
+                                    }
+                                }
+                            } else {
+                                if (result instanceof HashMap) {
+                                    addEmittedValue(new Tuple(((HashMap<?, ?>) result).get("lat"), ((HashMap<?, ?>) result).get("lon")));
+                                } else {
+                                    addEmittedValue(result);
+                                }
+                            }
+                        }
+                    };
+                }
+            };
+            return context.factoryClazz.cast(factory);
         }
+
         ContextCompiler compiler = contexts.get(context);
         if (compiler != null) {
             return context.factoryClazz.cast(compiler.compile(script::apply, params));
diff --git a/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java b/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java
index 4eb49ebb42241..02e5d22e147d5 100644
--- a/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/search/aggregations/AggregatorTestCase.java
@@ -95,8 +95,10 @@
 import org.opensearch.index.fielddata.IndexFieldDataService;
 import org.opensearch.index.mapper.BinaryFieldMapper;
 import org.opensearch.index.mapper.CompletionFieldMapper;
+import org.opensearch.index.mapper.ConstantKeywordFieldMapper;
 import org.opensearch.index.mapper.ContentPath;
 import org.opensearch.index.mapper.DateFieldMapper;
+import org.opensearch.index.mapper.DerivedFieldMapper;
 import org.opensearch.index.mapper.FieldAliasMapper;
 import org.opensearch.index.mapper.FieldMapper;
 import org.opensearch.index.mapper.GeoPointFieldMapper;
@@ -124,7 +126,6 @@
 import org.opensearch.search.aggregations.AggregatorFactories.Builder;
 import org.opensearch.search.aggregations.MultiBucketConsumerService.MultiBucketConsumer;
 import org.opensearch.search.aggregations.bucket.nested.NestedAggregationBuilder;
-import org.opensearch.search.aggregations.bucket.terms.TermsAggregator;
 import org.opensearch.search.aggregations.metrics.MetricsAggregator;
 import org.opensearch.search.aggregations.pipeline.PipelineAggregator;
 import org.opensearch.search.aggregations.pipeline.PipelineAggregator.PipelineTree;
@@ -198,6 +199,7 @@ public abstract class AggregatorTestCase extends OpenSearchTestCase {
         denylist.add(ObjectMapper.NESTED_CONTENT_TYPE); // TODO support for nested
         denylist.add(CompletionFieldMapper.CONTENT_TYPE); // TODO support completion
         denylist.add(FieldAliasMapper.CONTENT_TYPE); // TODO support alias
+        denylist.add(DerivedFieldMapper.CONTENT_TYPE); // TODO support derived fields
         TYPE_TEST_DENYLIST = denylist;
     }
 
@@ -407,6 +409,7 @@ public boolean shouldCache(Query query) {
         );
         fieldNameToType.putAll(getFieldAliases(fieldTypes));
 
+        when(searchContext.maxAggRewriteFilters()).thenReturn(10_000);
         registerFieldTypes(searchContext, mapperService, fieldNameToType);
         doAnswer(invocation -> {
             /* Store the release-ables so we can release them at the end of the test case. This is important because aggregations don't
@@ -778,6 +781,10 @@ public void testSupportedFieldTypes() throws IOException {
                 source.put("doc_values", "true");
             }
 
+            if (mappedType.getKey().equals(ConstantKeywordFieldMapper.CONTENT_TYPE) == true) {
+                source.put("value", "default_value");
+            }
+
             Mapper.Builder builder = mappedType.getValue().parse(fieldName, source, new MockParserContext());
             FieldMapper mapper = (FieldMapper) builder.build(new BuilderContext(settings, new ContentPath()));
 
@@ -1116,7 +1123,7 @@ protected static class CountingAggregator extends Aggregator {
         private final AtomicInteger collectCounter;
         public final Aggregator delegate;
 
-        public CountingAggregator(AtomicInteger collectCounter, TermsAggregator delegate) {
+        public CountingAggregator(AtomicInteger collectCounter, Aggregator delegate) {
             this.collectCounter = collectCounter;
             this.delegate = delegate;
         }
diff --git a/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java b/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java
index 0ee889af5ce1a..ce76914882150 100644
--- a/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/snapshots/AbstractSnapshotIntegTestCase.java
@@ -101,6 +101,8 @@
 import java.util.function.Function;
 import java.util.function.Predicate;
 
+import static org.opensearch.index.remote.RemoteStoreEnums.DataCategory.SEGMENTS;
+import static org.opensearch.index.remote.RemoteStoreEnums.DataType.LOCK_FILES;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
@@ -559,19 +561,16 @@ protected void assertDocCount(String index, long count) {
     }
 
     protected String[] getLockFilesInRemoteStore(String remoteStoreIndex, String remoteStoreRepositoryName) throws IOException {
-        String indexUUID = client().admin()
-            .indices()
-            .prepareGetSettings(remoteStoreIndex)
-            .get()
-            .getSetting(remoteStoreIndex, IndexMetadata.SETTING_INDEX_UUID);
-        return getLockFilesInRemoteStore(remoteStoreIndex, remoteStoreRepositoryName, indexUUID);
-    }
-
-    protected String[] getLockFilesInRemoteStore(String remoteStoreIndex, String remoteStoreRepositoryName, String indexUUID)
-        throws IOException {
         final RepositoriesService repositoriesService = internalCluster().getCurrentClusterManagerNodeInstance(RepositoriesService.class);
         final BlobStoreRepository remoteStoreRepository = (BlobStoreRepository) repositoriesService.repository(remoteStoreRepositoryName);
-        BlobPath shardLevelBlobPath = remoteStoreRepository.basePath().add(indexUUID).add("0").add("segments").add("lock_files");
+        BlobPath shardLevelBlobPath = getShardLevelBlobPath(
+            client(),
+            remoteStoreIndex,
+            remoteStoreRepository.basePath(),
+            "0",
+            SEGMENTS,
+            LOCK_FILES
+        );
         BlobContainer blobContainer = remoteStoreRepository.blobStore().blobContainer(shardLevelBlobPath);
         try (RemoteBufferedOutputDirectory lockDirectory = new RemoteBufferedOutputDirectory(blobContainer)) {
             return lockDirectory.listAll();
diff --git a/test/framework/src/main/java/org/opensearch/test/ClusterServiceUtils.java b/test/framework/src/main/java/org/opensearch/test/ClusterServiceUtils.java
index bb0c5f67179dc..f0c0e9bc2d589 100644
--- a/test/framework/src/main/java/org/opensearch/test/ClusterServiceUtils.java
+++ b/test/framework/src/main/java/org/opensearch/test/ClusterServiceUtils.java
@@ -34,6 +34,7 @@
 import org.apache.logging.log4j.core.util.Throwables;
 import org.opensearch.OpenSearchException;
 import org.opensearch.Version;
+import org.opensearch.cluster.ClusterManagerMetrics;
 import org.opensearch.cluster.ClusterName;
 import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.ClusterStateUpdateTask;
@@ -69,7 +70,7 @@ public static ClusterManagerService createClusterManagerService(ThreadPool threa
             Settings.builder().put(Node.NODE_NAME_SETTING.getKey(), "test_cluster_manager_node").build(),
             new ClusterSettings(Settings.EMPTY, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS),
             threadPool,
-            NoopMetricsRegistry.INSTANCE
+            new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE)
         );
         AtomicReference<ClusterState> clusterStateRef = new AtomicReference<>(initialClusterState);
         clusterManagerService.setClusterStatePublisher((event, publishListener, ackListener) -> {
@@ -182,7 +183,12 @@ public static ClusterService createClusterService(
         MetricsRegistry metricsRegistry
     ) {
         Settings settings = Settings.builder().put("node.name", "test").put("cluster.name", "ClusterServiceTests").build();
-        ClusterService clusterService = new ClusterService(settings, clusterSettings, threadPool, metricsRegistry);
+        ClusterService clusterService = new ClusterService(
+            settings,
+            clusterSettings,
+            threadPool,
+            new ClusterManagerMetrics(metricsRegistry)
+        );
         clusterService.setNodeConnectionsService(createNoOpNodeConnectionsService());
         ClusterState initialClusterState = ClusterState.builder(new ClusterName(ClusterServiceUtils.class.getSimpleName()))
             .nodes(DiscoveryNodes.builder().add(localNode).localNodeId(localNode.getId()).clusterManagerNodeId(localNode.getId()))
@@ -197,7 +203,7 @@ public static ClusterService createClusterService(
     }
 
     public static ClusterService createClusterService(Settings settings, ClusterSettings clusterSettings, ThreadPool threadPool) {
-        return new ClusterService(settings, clusterSettings, threadPool, NoopMetricsRegistry.INSTANCE);
+        return new ClusterService(settings, clusterSettings, threadPool, new ClusterManagerMetrics(NoopMetricsRegistry.INSTANCE));
     }
 
     public static NodeConnectionsService createNoOpNodeConnectionsService() {
diff --git a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java
index c2b964aa96212..ca80c65e58522 100644
--- a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java
+++ b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java
@@ -2736,6 +2736,7 @@ public void ensureEstimatedStats() {
                     false,
                     false,
                     false,
+                    false,
                     false
                 );
                 assertThat(
diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java
index 7cb1b3f4fe0d8..a9f6fdc86155d 100644
--- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java
@@ -96,6 +96,7 @@
 import org.opensearch.common.concurrent.GatedCloseable;
 import org.opensearch.common.network.NetworkModule;
 import org.opensearch.common.regex.Regex;
+import org.opensearch.common.settings.ClusterSettings;
 import org.opensearch.common.settings.FeatureFlagSettings;
 import org.opensearch.common.settings.Setting;
 import org.opensearch.common.settings.Setting.Property;
@@ -135,16 +136,19 @@
 import org.opensearch.index.engine.Segment;
 import org.opensearch.index.mapper.CompletionFieldMapper;
 import org.opensearch.index.mapper.MockFieldFilterPlugin;
+import org.opensearch.index.remote.RemoteStoreEnums.PathType;
 import org.opensearch.index.shard.IndexShard;
 import org.opensearch.index.store.Store;
 import org.opensearch.index.translog.Translog;
 import org.opensearch.indices.IndicesQueryCache;
 import org.opensearch.indices.IndicesRequestCache;
 import org.opensearch.indices.IndicesService;
+import org.opensearch.indices.RemoteStoreSettings;
 import org.opensearch.indices.replication.common.ReplicationType;
 import org.opensearch.indices.store.IndicesStore;
 import org.opensearch.monitor.os.OsInfo;
 import org.opensearch.node.NodeMocksPlugin;
+import org.opensearch.node.remotestore.RemoteStoreNodeService;
 import org.opensearch.plugins.NetworkPlugin;
 import org.opensearch.plugins.Plugin;
 import org.opensearch.repositories.fs.ReloadableFsRepository;
@@ -216,6 +220,8 @@
 import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_TYPE_ATTRIBUTE_KEY_FORMAT;
 import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_SEGMENT_REPOSITORY_NAME_ATTRIBUTE_KEY;
 import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_TRANSLOG_REPOSITORY_NAME_ATTRIBUTE_KEY;
+import static org.opensearch.node.remotestore.RemoteStoreNodeService.MIGRATION_DIRECTION_SETTING;
+import static org.opensearch.node.remotestore.RemoteStoreNodeService.REMOTE_STORE_COMPATIBILITY_MODE_SETTING;
 import static org.opensearch.test.XContentTestUtils.convertToMap;
 import static org.opensearch.test.XContentTestUtils.differenceBetweenMapsIgnoringArrayOrder;
 import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
@@ -377,6 +383,14 @@ public abstract class OpenSearchIntegTestCase extends OpenSearchTestCase {
      */
     public static final String TESTS_CLUSTER_NAME = "tests.clustername";
 
+    protected static final String REMOTE_BACKED_STORAGE_REPOSITORY_NAME = "test-remote-store-repo";
+
+    private Path remoteStoreRepositoryPath;
+
+    private ReplicationType randomReplicationType;
+
+    private String randomStorageType;
+
     @BeforeClass
     public static void beforeClass() throws Exception {
         testClusterRule.beforeClass();
@@ -1894,11 +1908,19 @@ protected Settings nodeSettings(int nodeOrdinal) {
             builder.put(TelemetrySettings.TRACER_ENABLED_SETTING.getKey(), true);
         }
 
-        // Randomly set a replication strategy for the node. Replication Strategy can still be manually overridden by subclass if needed.
+        // Randomly set a Replication Strategy and storage type for the node. Both Replication Strategy and Storage Type can still be
+        // manually overridden by subclass if needed.
         if (useRandomReplicationStrategy()) {
-            ReplicationType replicationType = randomBoolean() ? ReplicationType.DOCUMENT : ReplicationType.SEGMENT;
-            logger.info("Randomly using Replication Strategy as {}.", replicationType.toString());
-            builder.put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), replicationType);
+            if (randomReplicationType.equals(ReplicationType.SEGMENT) && randomStorageType.equals("REMOTE_STORE")) {
+                logger.info("Randomly using Replication Strategy as {} and Storage Type as {}.", randomReplicationType, randomStorageType);
+                if (remoteStoreRepositoryPath == null) {
+                    remoteStoreRepositoryPath = randomRepoPath().toAbsolutePath();
+                }
+                builder.put(remoteStoreClusterSettings(REMOTE_BACKED_STORAGE_REPOSITORY_NAME, remoteStoreRepositoryPath));
+            } else {
+                logger.info("Randomly using Replication Strategy as {} and Storage Type as {}.", randomReplicationType, randomStorageType);
+                builder.put(CLUSTER_REPLICATION_TYPE_SETTING.getKey(), randomReplicationType);
+            }
         }
         return builder.build();
     }
@@ -1951,6 +1973,14 @@ protected boolean ignoreExternalCluster() {
     }
 
     protected TestCluster buildTestCluster(Scope scope, long seed) throws IOException {
+        if (useRandomReplicationStrategy()) {
+            randomReplicationType = randomBoolean() ? ReplicationType.DOCUMENT : ReplicationType.SEGMENT;
+            if (randomReplicationType.equals(ReplicationType.SEGMENT)) {
+                randomStorageType = randomBoolean() ? "REMOTE_STORE" : "LOCAL";
+            } else {
+                randomStorageType = "LOCAL";
+            }
+        }
         String clusterAddresses = System.getProperty(TESTS_CLUSTER);
         if (Strings.hasLength(clusterAddresses) && ignoreExternalCluster() == false) {
             if (scope == Scope.TEST) {
@@ -2379,6 +2409,12 @@ protected String primaryNodeName(String indexName) {
         return clusterState.getRoutingNodes().node(nodeId).node().getName();
     }
 
+    protected String primaryNodeName(String indexName, int shardId) {
+        ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
+        String nodeId = clusterState.getRoutingTable().index(indexName).shard(shardId).primaryShard().currentNodeId();
+        return clusterState.getRoutingNodes().node(nodeId).node().getName();
+    }
+
     protected String replicaNodeName(String indexName) {
         ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
         String nodeId = clusterState.getRoutingTable().index(indexName).shard(0).replicaShards().get(0).currentNodeId();
@@ -2400,6 +2436,15 @@ protected RefreshResponse refreshAndWaitForReplication(String... indices) {
         return refreshResponse;
     }
 
+    public boolean isMigratingToRemoteStore() {
+        ClusterSettings clusterSettings = clusterService().getClusterSettings();
+        boolean isMixedMode = clusterSettings.get(REMOTE_STORE_COMPATIBILITY_MODE_SETTING)
+            .equals(RemoteStoreNodeService.CompatibilityMode.MIXED);
+        boolean isRemoteStoreMigrationDirection = clusterSettings.get(MIGRATION_DIRECTION_SETTING)
+            .equals(RemoteStoreNodeService.Direction.REMOTE_STORE);
+        return (isMixedMode && isRemoteStoreMigrationDirection);
+    }
+
     /**
      * Waits until active/started replica shards are caught up with primary shard only when Segment Replication is enabled.
      * This doesn't wait for inactive/non-started replica shards to become active/started.
@@ -2424,11 +2469,13 @@ protected void waitForReplication(String... indices) {
                                             for (ShardRouting replica : replicaRouting) {
                                                 if (replica.state().toString().equals("STARTED")) {
                                                     IndexShard replicaShard = getIndexShard(replica, index);
-                                                    assertEquals(
-                                                        "replica shards haven't caught up with primary",
-                                                        getLatestSegmentInfoVersion(primaryShard),
-                                                        getLatestSegmentInfoVersion(replicaShard)
-                                                    );
+                                                    if (replicaShard.indexSettings().isSegRepEnabledOrRemoteNode()) {
+                                                        assertEquals(
+                                                            "replica shards haven't caught up with primary",
+                                                            getLatestSegmentInfoVersion(primaryShard),
+                                                            getLatestSegmentInfoVersion(replicaShard)
+                                                        );
+                                                    }
                                                 }
                                             }
                                         }
@@ -2452,7 +2499,7 @@ protected void waitForReplication(String... indices) {
      * Checks if Segment Replication is enabled on Index.
      */
     protected boolean isSegmentReplicationEnabledForIndex(String index) {
-        return clusterService().state().getMetadata().isSegmentReplicationEnabled(index);
+        return clusterService().state().getMetadata().isSegmentReplicationEnabled(index) || isMigratingToRemoteStore();
     }
 
     protected IndexShard getIndexShard(ShardRouting routing, String indexName) {
@@ -2537,7 +2584,7 @@ public static Settings buildRemoteStoreNodeAttributes(
         );
     }
 
-    public static Settings buildRemoteStoreNodeAttributes(
+    private static Settings buildRemoteStoreNodeAttributes(
         String segmentRepoName,
         Path segmentRepoPath,
         String segmentRepoType,
@@ -2592,6 +2639,7 @@ public static Settings buildRemoteStoreNodeAttributes(
             settings.put(segmentRepoSettingsAttributeKeyPrefix + "compress", randomBoolean())
                 .put(segmentRepoSettingsAttributeKeyPrefix + "chunk_size", 200, ByteSizeUnit.BYTES);
         }
+        settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PATH_TYPE_SETTING.getKey(), randomFrom(PathType.values()));
         return settings.build();
     }
 
diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java
index aac3fca9e1e16..5a3f3b5a07a8d 100644
--- a/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java
+++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchTestCase.java
@@ -63,14 +63,17 @@
 import org.apache.lucene.tests.util.TimeUnits;
 import org.opensearch.Version;
 import org.opensearch.bootstrap.BootstrapForTesting;
+import org.opensearch.client.Client;
 import org.opensearch.client.Requests;
 import org.opensearch.cluster.ClusterModule;
+import org.opensearch.cluster.ClusterState;
 import org.opensearch.cluster.coordination.PersistedStateRegistry;
 import org.opensearch.cluster.metadata.IndexMetadata;
 import org.opensearch.cluster.node.DiscoveryNode;
 import org.opensearch.common.CheckedRunnable;
 import org.opensearch.common.Numbers;
 import org.opensearch.common.SuppressForbidden;
+import org.opensearch.common.blobstore.BlobPath;
 import org.opensearch.common.io.PathUtils;
 import org.opensearch.common.io.PathUtilsForTesting;
 import org.opensearch.common.io.stream.BytesStreamOutput;
@@ -120,6 +123,8 @@
 import org.opensearch.index.analysis.NamedAnalyzer;
 import org.opensearch.index.analysis.TokenFilterFactory;
 import org.opensearch.index.analysis.TokenizerFactory;
+import org.opensearch.index.remote.RemoteStoreEnums;
+import org.opensearch.index.remote.RemoteStorePathStrategy;
 import org.opensearch.index.store.remote.filecache.FileCache;
 import org.opensearch.indices.analysis.AnalysisModule;
 import org.opensearch.monitor.jvm.JvmInfo;
@@ -211,7 +216,12 @@
     "LuceneFixedGap",
     "LuceneVarGapFixedInterval",
     "LuceneVarGapDocFreqInterval",
-    "Lucene50" })
+    "Lucene50",
+    "Lucene90",
+    "Lucene94",
+    "Lucene90",
+    "Lucene95",
+    "Lucene99" })
 @LuceneTestCase.SuppressReproduceLine
 public abstract class OpenSearchTestCase extends LuceneTestCase {
 
@@ -1792,4 +1802,39 @@ protected static InetAddress randomIp(boolean v4) {
             throw new AssertionError();
         }
     }
+
+    public static BlobPath getShardLevelBlobPath(
+        Client client,
+        String remoteStoreIndex,
+        BlobPath basePath,
+        String shardId,
+        RemoteStoreEnums.DataCategory dataCategory,
+        RemoteStoreEnums.DataType dataType
+    ) {
+        String indexUUID = client.admin()
+            .indices()
+            .prepareGetSettings(remoteStoreIndex)
+            .get()
+            .getSetting(remoteStoreIndex, IndexMetadata.SETTING_INDEX_UUID);
+        ClusterState state = client.admin().cluster().prepareState().execute().actionGet().getState();
+        Map<String, String> remoteCustomData = state.metadata()
+            .index(remoteStoreIndex)
+            .getCustomData(IndexMetadata.REMOTE_STORE_CUSTOM_KEY);
+        RemoteStoreEnums.PathType type = Objects.isNull(remoteCustomData)
+            ? RemoteStoreEnums.PathType.FIXED
+            : RemoteStoreEnums.PathType.valueOf(remoteCustomData.get(RemoteStoreEnums.PathType.NAME));
+        RemoteStoreEnums.PathHashAlgorithm hashAlgorithm = Objects.nonNull(remoteCustomData)
+            ? remoteCustomData.containsKey(RemoteStoreEnums.PathHashAlgorithm.NAME)
+                ? RemoteStoreEnums.PathHashAlgorithm.valueOf(remoteCustomData.get(RemoteStoreEnums.PathHashAlgorithm.NAME))
+                : null
+            : null;
+        RemoteStorePathStrategy.PathInput pathInput = RemoteStorePathStrategy.PathInput.builder()
+            .basePath(basePath)
+            .indexUUID(indexUUID)
+            .shardId(shardId)
+            .dataCategory(dataCategory)
+            .dataType(dataType)
+            .build();
+        return type.path(pathInput, hashAlgorithm);
+    }
 }
diff --git a/test/framework/src/main/java/org/opensearch/test/gateway/TestGatewayAllocator.java b/test/framework/src/main/java/org/opensearch/test/gateway/TestGatewayAllocator.java
index b1695ff00e0cc..b9f52a62f823a 100644
--- a/test/framework/src/main/java/org/opensearch/test/gateway/TestGatewayAllocator.java
+++ b/test/framework/src/main/java/org/opensearch/test/gateway/TestGatewayAllocator.java
@@ -183,4 +183,5 @@ public String getReplicationCheckPointKey(ShardId shardId, String nodeName) {
     public void addReplicationCheckpoint(ShardId shardId, String nodeName, ReplicationCheckpoint replicationCheckpoint) {
         shardIdNodeToReplicationCheckPointMap.putIfAbsent(getReplicationCheckPointKey(shardId, nodeName), replicationCheckpoint);
     }
+
 }
diff --git a/test/framework/src/main/java/org/opensearch/test/gateway/TestShardBatchGatewayAllocator.java b/test/framework/src/main/java/org/opensearch/test/gateway/TestShardBatchGatewayAllocator.java
new file mode 100644
index 0000000000000..53a4e90adb976
--- /dev/null
+++ b/test/framework/src/main/java/org/opensearch/test/gateway/TestShardBatchGatewayAllocator.java
@@ -0,0 +1,144 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.test.gateway;
+
+import org.opensearch.cluster.node.DiscoveryNode;
+import org.opensearch.cluster.node.DiscoveryNodes;
+import org.opensearch.cluster.routing.ShardRouting;
+import org.opensearch.cluster.routing.allocation.AllocateUnassignedDecision;
+import org.opensearch.cluster.routing.allocation.RoutingAllocation;
+import org.opensearch.core.index.shard.ShardId;
+import org.opensearch.gateway.AsyncShardFetch;
+import org.opensearch.gateway.PrimaryShardBatchAllocator;
+import org.opensearch.gateway.ReplicaShardBatchAllocator;
+import org.opensearch.gateway.ShardsBatchGatewayAllocator;
+import org.opensearch.gateway.TransportNodesGatewayStartedShardHelper;
+import org.opensearch.gateway.TransportNodesListGatewayStartedShardsBatch;
+import org.opensearch.indices.replication.checkpoint.ReplicationCheckpoint;
+import org.opensearch.indices.store.TransportNodesListShardStoreMetadataBatch;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class TestShardBatchGatewayAllocator extends ShardsBatchGatewayAllocator {
+
+    Map<String /* node id */, Map<ShardId, ShardRouting>> knownAllocations = new HashMap<>();
+    DiscoveryNodes currentNodes = DiscoveryNodes.EMPTY_NODES;
+    Map<String, ReplicationCheckpoint> shardIdNodeToReplicationCheckPointMap = new HashMap<>();
+
+    PrimaryShardBatchAllocator primaryBatchShardAllocator = new PrimaryShardBatchAllocator() {
+        @Override
+        protected AsyncShardFetch.FetchResult<TransportNodesListGatewayStartedShardsBatch.NodeGatewayStartedShardsBatch> fetchData(
+            List<ShardRouting> eligibleShards,
+            List<ShardRouting> inEligibleShards,
+            RoutingAllocation allocation
+        ) {
+            Map<DiscoveryNode, TransportNodesListGatewayStartedShardsBatch.NodeGatewayStartedShardsBatch> foundShards = new HashMap<>();
+            HashMap<ShardId, Set<String>> shardsToIgnoreNodes = new HashMap<>();
+            for (Map.Entry<String, Map<ShardId, ShardRouting>> entry : knownAllocations.entrySet()) {
+                String nodeId = entry.getKey();
+                Map<ShardId, ShardRouting> shardsOnNode = entry.getValue();
+                HashMap<ShardId, TransportNodesGatewayStartedShardHelper.GatewayStartedShard> adaptedResponse = new HashMap<>();
+
+                for (ShardRouting shardRouting : eligibleShards) {
+                    ShardId shardId = shardRouting.shardId();
+                    Set<String> ignoreNodes = allocation.getIgnoreNodes(shardId);
+
+                    if (shardsOnNode.containsKey(shardId) && ignoreNodes.contains(nodeId) == false && currentNodes.nodeExists(nodeId)) {
+                        TransportNodesGatewayStartedShardHelper.GatewayStartedShard nodeShard =
+                            new TransportNodesGatewayStartedShardHelper.GatewayStartedShard(
+                                shardsOnNode.get(shardId).allocationId().getId(),
+                                shardsOnNode.get(shardId).primary(),
+                                getReplicationCheckpoint(shardId, nodeId)
+                            );
+                        adaptedResponse.put(shardId, nodeShard);
+                        shardsToIgnoreNodes.put(shardId, ignoreNodes);
+                    }
+                    foundShards.put(
+                        currentNodes.get(nodeId),
+                        new TransportNodesListGatewayStartedShardsBatch.NodeGatewayStartedShardsBatch(
+                            currentNodes.get(nodeId),
+                            adaptedResponse
+                        )
+                    );
+                }
+            }
+            return new AsyncShardFetch.FetchResult<>(foundShards, shardsToIgnoreNodes);
+        }
+    };
+
+    ReplicaShardBatchAllocator replicaBatchShardAllocator = new ReplicaShardBatchAllocator() {
+
+        @Override
+        protected AsyncShardFetch.FetchResult<TransportNodesListShardStoreMetadataBatch.NodeStoreFilesMetadataBatch> fetchData(
+            List<ShardRouting> eligibleShards,
+            List<ShardRouting> inEligibleShards,
+            RoutingAllocation allocation
+        ) {
+            return new AsyncShardFetch.FetchResult<>(Collections.emptyMap(), Collections.emptyMap());
+        }
+
+        @Override
+        protected boolean hasInitiatedFetching(ShardRouting shard) {
+            return true;
+        }
+    };
+
+    @Override
+    public void allocateAllUnassignedShards(RoutingAllocation allocation, boolean primary) {
+        currentNodes = allocation.nodes();
+        innerAllocateUnassignedBatch(allocation, primaryBatchShardAllocator, replicaBatchShardAllocator, primary);
+    }
+
+    @Override
+    public void beforeAllocation(RoutingAllocation allocation) {}
+
+    @Override
+    public void afterPrimariesBeforeReplicas(RoutingAllocation allocation) {}
+
+    public Set<String> createAndUpdateBatches(RoutingAllocation allocation, boolean primary) {
+        return super.createAndUpdateBatches(allocation, primary);
+    }
+
+    public void safelyRemoveShardFromBatch(ShardRouting shard) {
+        super.safelyRemoveShardFromBatch(shard, shard.primary());
+    }
+
+    public void safelyRemoveShardFromBothBatch(ShardRouting shardRouting) {
+        super.safelyRemoveShardFromBothBatch(shardRouting);
+    }
+
+    public String getBatchId(ShardRouting shard, boolean primary) {
+        return super.getBatchId(shard, primary);
+    }
+
+    public Map<String, ShardsBatchGatewayAllocator.ShardsBatch> getBatchIdToStartedShardBatch() {
+        return batchIdToStartedShardBatch;
+    }
+
+    public Map<String, ShardsBatchGatewayAllocator.ShardsBatch> getBatchIdToStoreShardBatch() {
+        return batchIdToStoreShardBatch;
+    }
+
+    @Override
+    public AllocateUnassignedDecision explainUnassignedShardAllocation(ShardRouting unassignedShard, RoutingAllocation routingAllocation) {
+        return super.explainUnassignedShardAllocation(unassignedShard, routingAllocation);
+    }
+
+    protected ReplicationCheckpoint getReplicationCheckpoint(ShardId shardId, String nodeName) {
+        return shardIdNodeToReplicationCheckPointMap.getOrDefault(getReplicationCheckPointKey(shardId, nodeName), null);
+    }
+
+    public String getReplicationCheckPointKey(ShardId shardId, String nodeName) {
+        return shardId.toString() + "_" + nodeName;
+    }
+}
diff --git a/test/framework/src/main/java/org/opensearch/transport/TestRequest.java b/test/framework/src/main/java/org/opensearch/transport/TestRequest.java
index 2fe917235e948..0bb5a6e16fff1 100644
--- a/test/framework/src/main/java/org/opensearch/transport/TestRequest.java
+++ b/test/framework/src/main/java/org/opensearch/transport/TestRequest.java
@@ -54,4 +54,8 @@ public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
         out.writeString(value);
     }
+
+    public String getValue() {
+        return value;
+    }
 }
diff --git a/test/framework/src/main/java/org/opensearch/transport/TestResponse.java b/test/framework/src/main/java/org/opensearch/transport/TestResponse.java
index 14db8b3372bf2..7fd7c760c9cf6 100644
--- a/test/framework/src/main/java/org/opensearch/transport/TestResponse.java
+++ b/test/framework/src/main/java/org/opensearch/transport/TestResponse.java
@@ -54,4 +54,8 @@ public TestResponse(StreamInput in) throws IOException {
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(value);
     }
+
+    public String getValue() {
+        return value;
+    }
 }
diff --git a/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/StrictCheckSpanProcessor.java b/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/StrictCheckSpanProcessor.java
index f7ebb3ee18a9b..4e72caeea584e 100644
--- a/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/StrictCheckSpanProcessor.java
+++ b/test/telemetry/src/main/java/org/opensearch/test/telemetry/tracing/StrictCheckSpanProcessor.java
@@ -8,6 +8,7 @@
 
 package org.opensearch.test.telemetry.tracing;
 
+import org.opensearch.common.Booleans;
 import org.opensearch.telemetry.tracing.Span;
 import org.opensearch.test.telemetry.tracing.validators.AllSpansAreEndedProperly;
 import org.opensearch.test.telemetry.tracing.validators.AllSpansHaveUniqueId;
@@ -29,6 +30,14 @@ public StrictCheckSpanProcessor() {}
 
     private static Map<String, MockSpanData> spanMap = new ConcurrentHashMap<>();
 
+    // If you want to see the stack trace for each spanData, then
+    // update the flag to true or set the corresponding system property to true
+    // This is helpful in debugging the tests. Default value is false.
+    // Note: Enabling this might lead to OOM issues while running ITs.
+    private static final boolean isStackTraceForSpanEnabled = Booleans.parseBoolean(
+        System.getProperty("tests.telemetry.span.stack_traces", "false")
+    );
+
     @Override
     public void onStart(Span span) {
         spanMap.put(span.getSpanId(), toMockSpanData(span));
@@ -53,6 +62,7 @@ public List<MockSpanData> getFinishedSpanItems() {
 
     private MockSpanData toMockSpanData(Span span) {
         String parentSpanId = (span.getParentSpan() != null) ? span.getParentSpan().getSpanId() : "";
+        StackTraceElement[] stackTrace = isStackTraceForSpanEnabled ? Thread.currentThread().getStackTrace() : null;
         MockSpanData spanData = new MockSpanData(
             span.getSpanId(),
             parentSpanId,
@@ -60,7 +70,7 @@ private MockSpanData toMockSpanData(Span span) {
             System.nanoTime(),
             false,
             span.getSpanName(),
-            Thread.currentThread().getStackTrace(),
+            stackTrace,
             (span instanceof MockSpan) ? ((MockSpan) span).getAttributes() : Map.of()
         );
         return spanData;