Merge branch 'main' into feature/master-task-throttling

.ci/bwcVersions

@@ -41,9 +41,12 @@ BWC_VERSION:
   - "1.3.2"
   - "1.3.3"
   - "1.3.4"
+  - "1.3.5"
   - "2.0.0"
   - "2.0.1"
   - "2.0.2"
   - "2.1.0"
   - "2.1.1"
   - "2.2.0"
+  - "2.2.1"
+  - "2.3.0"

.github/pull_request_template.md

@@ -1,15 +1,16 @@
 ### Description
 [Describe what this change achieves]
- 
+
 ### Issues Resolved
 [List any issues this PR will resolve]
- 
+
 ### Check List
 - [ ] New functionality includes testing.
   - [ ] All tests pass
 - [ ] New functionality has been documented.
   - [ ] New functionality has javadoc added
-- [ ] Commits are signed per the DCO using --signoff 
+- [ ] Commits are signed per the DCO using --signoff
+- [ ] Commit changes are listed out in CHANGELOG.md file (See: [Changelog](../CONTRIBUTING.md#changelog))
 
 By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
 For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin).

.github/workflows/auto-release.yml

@@ -3,7 +3,7 @@ name: Releases
 on:
   push:
     tags:
-      - '*.*.*'
+      - '*'
 
 jobs:
 
@@ -12,11 +12,18 @@ jobs:
     permissions:
       contents: write
     steps:
+      - name: GitHub App token
+        id: github_app_token
+        uses: tibdex/[email protected]
+        with:
+          app_id: ${{ secrets.APP_ID }}
+          private_key: ${{ secrets.APP_PRIVATE_KEY }}
+          installation_id: 22958780
       - name: Get tag
         id: tag
         uses: dawidd6/action-get-tag@v1
       - uses: actions/checkout@v2
       - uses: ncipollo/release-action@v1
         with:
-          token: ${{ secrets.GITHUB_TOKEN }}
+          github_token: ${{ steps.github_app_token.outputs.token }}
           bodyFile: release-notes/opensearch.release-notes-${{steps.tag.outputs.tag}}.md

.github/workflows/changelog_verifier.yml

@@ -0,0 +1,22 @@
+name: "Changelog Verifier"
+on:
+  pull_request:
+    types: [opened, edited, review_requested, synchronize, reopened, ready_for_review, labeled, unlabeled]
+
+jobs:
+  # Enforces the update of a changelog file on every pull request
+  verify-changelog:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          ref: ${{ github.event.pull_request.head.ref }}
+
+      - uses: dangoslen/dependabot-changelog-helper@v1
+
+      - uses: stefanzweifel/git-auto-commit-action@v4
+        with:
+          commit_message: "Update changelog"
+
+      - uses: dangoslen/changelog-enforcer@v3

.github/workflows/gradle-check.yml

@@ -2,9 +2,9 @@ name: Gradle Check (Jenkins)
 on:
   push:
     branches-ignore:
-      - 'backport/*'
-      - 'create-pull-request/*'
-      - 'dependabot/*'
+      - 'backport/**'
+      - 'create-pull-request/**'
+      - 'dependabot/**'
   pull_request_target:
     types: [opened, synchronize, reopened]
 

CHANGELOG.md

@@ -0,0 +1,19 @@
+# CHANGELOG
+Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
+
+## [Unreleased]
+### Added
+- Github workflow for changelog verification ([#4085](https://github.com/opensearch-project/OpenSearch/pull/4085))
+
+### Changed
+
+### Deprecated
+
+### Removed
+
+### Fixed
+
+### Security
+
+
+[Unreleased]: https://github.com/opensearch-project/OpenSearch/compare/2.2.0...HEAD

CONTRIBUTING.md

@@ -6,6 +6,7 @@
     - [Documentation Changes](#documentation-changes)
     - [Contributing Code](#contributing-code)
   - [Developer Certificate of Origin](#developer-certificate-of-origin)
+  - [Changelog](#changelog)
   - [Review Process](#review-process)
 
 # Contributing to OpenSearch
@@ -59,6 +60,16 @@ If you would like to contribute to the documentation, please do so in the [docum
 
 As with other types of contributions, the first step is to [**open an issue on GitHub**](https://github.com/opensearch-project/OpenSearch/issues/new/choose). Opening an issue before you make changes makes sure that someone else isn't already working on that particular problem. It also lets us all work together to find the right approach before you spend a bunch of time on a PR. So again, when in doubt, open an issue.
 
+Additionally, here are a few guidelines to help you decide whether a particular feature should be included in OpenSearch.
+
+**Is your feature important to most users of OpenSearch?**
+
+If you believe that a feature is going to fulfill a need for most users of OpenSearch, then it belongs in OpenSearch. However, we don't want every feature built into the core server. If the feature requires additional permissions or brings in extra dependencies it should instead be included as a module in core.
+
+**Is your feature a common dependency across multiple plugins?**
+
+Does this feature contain functionality that cuts across multiple plugins? If so, this most likely belongs in OpenSearch as a core module or plugin.
+
 Once you've opened an issue, check out our [Developer Guide](./DEVELOPER_GUIDE.md) for instructions on how to get started.
 
 ## Developer Certificate of Origin
@@ -106,6 +117,25 @@ Signed-off-by: Jane Smith <[email protected]>
 ```
 You may type this line on your own when writing your commit messages. However, if your user.name and user.email are set in your git configs, you can use `-s` or `--signoff` to add the `Signed-off-by` line to the end of the commit message.
 
+## Changelog
+
+OpenSearch maintains version specific changelog by enforcing a change to the ongoing [CHANGELOG](CHANGELOG.md) file adhering to the [Keep A Changelog](https://keepachangelog.com/en/1.0.0/) format.
+
+Briefly, the changes are curated by version, with the changes to the main branch added chronologically to `Unreleased` version. Further, each version has corresponding sections which list out the category of the change - `Added`, `Changed`, `Deprecated`, `Removed`, `Fixed`, `Security`.
+
+
+### How to add my changes to [CHANGELOG](CHANGELOG.md)?
+
+As a contributor, you must ensure that every pull request has the changes listed out within the corresponding version and appropriate section of [CHANGELOG](CHANGELOG.md) file.
+
+Adding in the change is two step process -
+1. Add your changes to the corresponding section within the CHANGELOG file with dummy pull request information, publish the PR
+
+  `Your change here ([#PR_NUMBER](PR_URL))`
+
+2. Update the entry for your change in [`CHANGELOG.md`](CHANGELOG.md) and make sure that you reference the pull request there.
+
+
 ## Review Process
 
 We deeply appreciate everyone who takes the time to make a contribution. We will review all contributions as quickly as possible. As a reminder, [opening an issue](https://github.com/opensearch-project/OpenSearch/issues/new/choose) discussing your change before you make it is the best way to smooth the PR process. This will prevent a rejection because someone else is already working on the problem, or because the solution is incompatible with the architectural direction.

DEVELOPER_GUIDE.md

@@ -4,14 +4,16 @@
     - [Install Prerequisites](#install-prerequisites)
       - [JDK 11](#jdk-11)
       - [JDK 14](#jdk-14)
-      - [Runtime JDK](#runtime-jdk)
+      - [JDK 17](#jdk-17)
+      - [Custom Runtime JDK](#custom-runtime-jdk)
       - [Windows](#windows)
       - [Docker](#docker)
     - [Build](#build)
     - [Run Tests](#run-tests)
     - [Run OpenSearch](#run-opensearch)
   - [Use an Editor](#use-an-editor)
     - [IntelliJ IDEA](#intellij-idea)
+      - [Remote development using JetBrains Gateway](#remote-development-using-jetbrains-gateway)
     - [Visual Studio Code](#visual-studio-code)
     - [Eclipse](#eclipse)
   - [Project Layout](#project-layout)
@@ -35,6 +37,7 @@
       - [testImplementation](#testimplementation)
     - [Gradle Plugins](#gradle-plugins)
       - [Distribution Download Plugin](#distribution-download-plugin)
+    - [Creating fat-JAR of a Module](#creating-fat-jar-of-a-module)
   - [Misc](#misc)
     - [git-secrets](#git-secrets)
       - [Installation](#installation)
@@ -49,7 +52,7 @@
   - [Submitting Changes](#submitting-changes)
   - [Backports](#backports)
   - [LineLint](#linelint)
-  - [Lucene Snapshots](#lucene-snapshots)
+- [Lucene Snapshots](#lucene-snapshots)
 
 # Developer Guide
 
@@ -374,6 +377,42 @@ The Distribution Download plugin downloads the latest version of OpenSearch by d
 ./gradlew integTest -PcustomDistributionUrl="https://ci.opensearch.org/ci/dbc/bundle-build/1.2.0/1127/linux/x64/dist/opensearch-1.2.0-linux-x64.tar.gz"
 ```
 
+### Creating fat-JAR of a Module
+
+A fat-JAR (or an uber-JAR) is the JAR, which contains classes from all the libraries, on which your project depends and, of course, the classes of current project.
+
+There might be cases where a developer would like to add some custom logic to the code of a module (or multiple modules) and generate a fat-JAR that can be directly used by the dependency management tool. For example, in [#3665](https://github.com/opensearch-project/OpenSearch/pull/3665) a developer wanted to provide a tentative patch as a fat-JAR to a consumer for changes made in the high level REST client. 
+
+Use [Gradle Shadow plugin](https://imperceptiblethoughts.com/shadow/).
+Add the following to the `build.gradle` file of the module for which you want to create the fat-JAR, e.g. `client/rest-high-level/build.gradle`:
+
+```
+apply plugin: 'com.github.johnrengelman.shadow'
+```
+
+Run the `shadowJar` command using:
+```
+./gradlew :client:rest-high-level:shadowJar
+```
+
+This will generate a fat-JAR in the `build/distributions` folder of the module, e.g. .`/client/rest-high-level/build/distributions/opensearch-rest-high-level-client-1.4.0-SNAPSHOT.jar`.
+
+You can further customize your fat-JAR by customising the plugin, More information about shadow plugin can be found [here](https://imperceptiblethoughts.com/shadow/).
+
+To use the generated JAR, install the JAR locally, e.g. 
+```
+mvn install:install-file -Dfile=src/main/resources/opensearch-rest-high-level-client-1.4.0-SNAPSHOT.jar -DgroupId=org.opensearch.client -DartifactId=opensearch-rest-high-level-client -Dversion=1.4.0-SNAPSHOT -Dpackaging=jar -DgeneratePom=true
+```
+
+Refer the installed JAR as any other maven artifact, e.g.
+
+```
+<dependency>
+    <groupId>org.opensearch.client</groupId>
+    <artifactId>opensearch-rest-high-level-client</artifactId>
+    <version>1.4.0-SNAPSHOT</version>
+</dependency>
+```
 
 ## Misc
 
@@ -460,7 +499,7 @@ Work to make sure that OpenSearch can scale in a distributed manner.
 
 Includes:
 
-- Nodes (Master, Data, Compute, Ingest, Discovery, etc.)
+- Nodes (Cluster Manager, Data, Compute, Ingest, Discovery, etc.)
 - Replication & Merge Policies (Document, Segment level)
 - Snapshot/Restore (repositories; S3, Azure, GCP, NFS)
 - Translog (e.g., OpenSearch, Kafka, Kinesis)

README.md

@@ -1,7 +1,13 @@
-[![codecov](https://codecov.io/gh/opensearch-project/OpenSearch/branch/main/graph/badge.svg)](https://codecov.io/gh/opensearch-project/OpenSearch)
-
 <img src="https://opensearch.org/assets/img/opensearch-logo-themed.svg" height="64px">
 
+[![Chat](https://img.shields.io/badge/chat-on%20forums-blue)](https://forum.opensearch.org/c/opensearch/)
+[![Documentation](https://img.shields.io/badge/documentation-reference-blue)](https://opensearch.org/docs/latest/opensearch/index/)
+[![codecov](https://codecov.io/gh/opensearch-project/OpenSearch/branch/main/graph/badge.svg)](https://codecov.io/gh/opensearch-project/OpenSearch)
+[![GHA gradle check](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/gradle-check.yml)
+[![GHA validate pull request](https://github.com/opensearch-project/OpenSearch/actions/workflows/wrapper.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/wrapper.yml)
+[![GHA precommit](https://github.com/opensearch-project/OpenSearch/actions/workflows/precommit.yml/badge.svg)](https://github.com/opensearch-project/OpenSearch/actions/workflows/precommit.yml)
+[![Jenkins gradle check job](https://img.shields.io/jenkins/build?jobUrl=https%3A%2F%2Fbuild.ci.opensearch.org%2Fjob%2Fgradle-check%2F&label=Jenkins%20Gradle%20Check)](https://build.ci.opensearch.org/job/gradle-check/)
+
 - [Welcome!](#welcome)
 - [Project Resources](#project-resources)
 - [Code of Conduct](#code-of-conduct)

TESTING.md

@@ -395,7 +395,7 @@ The branch needs to be available on the remote that the BWC makes of the reposit
 
 Example:
 
-Say you need to make a change to `master` and have a BWC layer in `5.x`. You will need to: . Create a branch called `index_req_change` off your remote `${remote}`. This will contain your change. . Create a branch called `index_req_bwc_5.x` off `5.x`. This will contain your bwc layer. . Push both branches to your remote repository. . Run the tests with `./gradlew check -Dbwc.remote=${remote} -Dbwc.refspec.5.x=index_req_bwc_5.x`.
+Say you need to make a change to `main` and have a BWC layer in `5.x`. You will need to: . Create a branch called `index_req_change` off your remote `${remote}`. This will contain your change. . Create a branch called `index_req_bwc_5.x` off `5.x`. This will contain your bwc layer. . Push both branches to your remote repository. . Run the tests with `./gradlew check -Dbwc.remote=${remote} -Dbwc.refspec.5.x=index_req_bwc_5.x`.
 
 ### Skip fetching latest
 

benchmarks/src/main/java/org/opensearch/benchmark/time/NanoTimeVsCurrentTimeMillisBenchmark.java

@@ -8,7 +8,15 @@
 
 package org.opensearch.benchmark.time;
 
-import org.openjdk.jmh.annotations.*;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Benchmark;
 
 import java.util.concurrent.TimeUnit;
 

build.gradle

@@ -28,6 +28,9 @@
  * under the License.
  */
 
+import java.nio.charset.StandardCharsets;
+import java.io.ByteArrayOutputStream;
+
 import com.avast.gradle.dockercompose.tasks.ComposePull
 import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin
 import de.thetaphi.forbiddenapis.gradle.ForbiddenApisPlugin
@@ -37,18 +40,22 @@ import org.opensearch.gradle.Version
 import org.opensearch.gradle.VersionProperties
 import org.opensearch.gradle.info.BuildParams
 import org.opensearch.gradle.plugin.PluginBuildPlugin
+import org.opensearch.gradle.tar.SymbolicLinkPreservingTar
 import org.gradle.plugins.ide.eclipse.model.AccessRule
 import org.gradle.plugins.ide.eclipse.model.EclipseJdt
 import org.gradle.plugins.ide.eclipse.model.SourceFolder
+import org.gradle.api.Project;
+import org.gradle.process.ExecResult;
 import org.gradle.util.DistributionLocator
 import org.gradle.util.GradleVersion
+
 import static org.opensearch.gradle.util.GradleUtils.maybeConfigure
 
 plugins {
   id 'lifecycle-base'
   id 'opensearch.docker-support'
   id 'opensearch.global-build-info'
-  id "com.diffplug.spotless" version "6.7.0" apply false
+  id "com.diffplug.spotless" version "6.9.1" apply false
   id "org.gradle.test-retry" version "1.4.0" apply false
   id "test-report-aggregation"
   id 'jacoco-report-aggregation'
@@ -150,6 +157,30 @@ Map<String, String> buildMetadataMap = buildMetadataValue.tokenize(';').collectE
   return [key, value]
 }
 
+    /**
+     * Using 'git' command line (if available), tries to fetch the commit date of the current revision
+     * @return commit date of the current revision or 0 if it is not available
+     */
+long gitRevisionDate = {
+  // Try to get last commit date as Unix timestamp
+  try (ByteArrayOutputStream stdout = new ByteArrayOutputStream()) {
+     ExecResult result = project.exec(spec -> {
+        spec.setIgnoreExitValue(true);
+        spec.setStandardOutput(stdout);
+        spec.commandLine("git", "log", "-1", "--format=%ct");
+     });
+
+    if (result.getExitValue() == 0) {
+      return Long.parseLong(stdout.toString(StandardCharsets.UTF_8).replaceAll("\\s", "")) * 1000; /* seconds to millis */
+    }
+  } catch (IOException | GradleException | NumberFormatException ex) {
+    /* fall back to default Unix epoch timestamp */
+  }
+
+  return 0;
+}()
+
+
 // injecting groovy property variables into all projects
 allprojects {
   project.ext {
@@ -282,11 +313,15 @@ allprojects {
   }
 
   // support for reproducible builds
-  tasks.withType(AbstractArchiveTask).configureEach {
+  tasks.withType(AbstractArchiveTask).configureEach { task ->
     // ignore file timestamps
     // be consistent in archive file order
-    preserveFileTimestamps = false
-    reproducibleFileOrder = true
+    task.preserveFileTimestamps = false
+    task.reproducibleFileOrder = true
+    if (task instanceof SymbolicLinkPreservingTar) {
+      // Replace file timestamps with latest Git revision date (if available)
+      task.lastModifiedTimestamp = gitRevisionDate
+    }
   }
 
   project.afterEvaluate {
@@ -324,7 +359,8 @@ allprojects {
           project.javadoc.dependsOn "${upstreamProject.path}:javadoc"
           String externalLinkName = upstreamProject.archivesBaseName
           String artifactPath = dep.group.replaceAll('\\.', '/') + '/' + externalLinkName.replaceAll('\\.', '/') + '/' + dep.version
-          project.javadoc.options.linksOffline artifactsHost + "/javadoc/" + artifactPath, "${upstreamProject.buildDir}/docs/javadoc/"
+          String projectRelativePath = project.relativePath(upstreamProject.buildDir)
+          project.javadoc.options.linksOffline artifactsHost + "/javadoc/" + artifactPath, "${projectRelativePath}/docs/javadoc/"
         }
       }
       boolean hasShadow = project.plugins.hasPlugin(ShadowPlugin)