Skip to content

Commit

Permalink
Merge remote-tracking branch 'upstream/main' into remote_global_state
Browse files Browse the repository at this point in the history
Signed-off-by: Dhwanil Patel <[email protected]>
  • Loading branch information
dhwanilpatel committed Oct 6, 2023
2 parents 3f92a88 + e4c477a commit d0d1ccf
Show file tree
Hide file tree
Showing 254 changed files with 5,271 additions and 1,209 deletions.
1 change: 1 addition & 0 deletions .github/pull_request_template.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ Resolves #[Issue number to be closed when this PR is merged]
- [ ] New functionality has javadoc added
- [ ] Commits are signed per the DCO using --signoff
- [ ] Commit changes are listed out in CHANGELOG.md file (See: [Changelog](../blob/main/CONTRIBUTING.md#changelog))
- [ ] Public documentation issue/PR [created](https://github.com/opensearch-project/documentation-website/issues/new/choose)

By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
For more information on following Developer Certificate of Origin and signing off your commits, please check [here](https://github.com/opensearch-project/OpenSearch/blob/main/CONTRIBUTING.md#developer-certificate-of-origin).
26 changes: 26 additions & 0 deletions .github/workflows/assemble.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
name: Gradle Assemble
on: [pull_request]

jobs:
assemble:
if: github.repository == 'opensearch-project/OpenSearch'
runs-on: ${{ matrix.os }}
strategy:
matrix:
os: [ubuntu-latest, windows-latest, macos-latest]
steps:
- uses: actions/checkout@v4
- name: Set up JDK 11
uses: actions/setup-java@v3
with:
java-version: 11
distribution: temurin
- name: Setup docker (missing on MacOS)
if: runner.os == 'macos'
run: |
brew install docker
colima start
sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock
- name: Run Gradle (assemble)
run: |
./gradlew assemble --parallel --no-build-cache -PDISABLE_BUILD_CACHE
2 changes: 1 addition & 1 deletion .github/workflows/lucene-snapshots.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ jobs:
- name: Set hash
working-directory: ./lucene
run: |
echo "::set-output name=REVISION::$(git rev-parse --short HEAD)"
echo "REVISION=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
id: version

- name: Initialize gradle settings
Expand Down
11 changes: 1 addition & 10 deletions .github/workflows/precommit.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: Gradle Precommit and Assemble
name: Gradle Precommit
on: [pull_request]

jobs:
Expand All @@ -19,12 +19,3 @@ jobs:
- name: Run Gradle (precommit)
run: |
./gradlew javadoc precommit --parallel
- name: Setup docker (missing on MacOS)
if: runner.os == 'macos'
run: |
brew install docker
colima start
sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock
- name: Run Gradle (assemble)
run: |
./gradlew assemble --parallel
6 changes: 3 additions & 3 deletions .github/workflows/version.yml
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ jobs:
sed -i "s/CURRENT = $CURRENT_VERSION_UNDERSCORE;/CURRENT = $NEXT_VERSION_UNDERSCORE;/g" libs/core/src/main/java/org/opensearch/Version.java
- name: Create Pull Request
uses: peter-evans/create-pull-request@v3
uses: peter-evans/create-pull-request@v5
with:
token: ${{ steps.github_app_token.outputs.token }}
base: ${{ env.BASE }}
Expand All @@ -86,7 +86,7 @@ jobs:
sed -i "s/public static final Version $CURRENT_VERSION_UNDERSCORE = new Version(\([[:digit:]]\+\)\(.*\));/\0\n public static final Version $NEXT_VERSION_UNDERSCORE = new Version($NEXT_VERSION_ID\2);/g" libs/core/src/main/java/org/opensearch/Version.java
- name: Create Pull Request
uses: peter-evans/create-pull-request@v3
uses: peter-evans/create-pull-request@v5
with:
token: ${{ steps.github_app_token.outputs.token }}
base: ${{ env.BASE_X }}
Expand All @@ -113,7 +113,7 @@ jobs:
sed -i "s/public static final Version $CURRENT_VERSION_UNDERSCORE = new Version(\([[:digit:]]\+\)\(.*\));/\0\n public static final Version $NEXT_VERSION_UNDERSCORE = new Version($NEXT_VERSION_ID\2);/g" libs/core/src/main/java/org/opensearch/Version.java
- name: Create Pull Request
uses: peter-evans/create-pull-request@v3
uses: peter-evans/create-pull-request@v5
with:
token: ${{ steps.github_app_token.outputs.token }}
base: main
Expand Down
25 changes: 20 additions & 5 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- Change http code on create index API with bad input raising NotXContentException from 500 to 400 ([#4773](https://github.com/opensearch-project/OpenSearch/pull/4773))
- Improve summary error message for invalid setting updates ([#4792](https://github.com/opensearch-project/OpenSearch/pull/4792))
- Return 409 Conflict HTTP status instead of 503 on failure to concurrently execute snapshots ([#8986](https://github.com/opensearch-project/OpenSearch/pull/5855))
- Performance improvement for Datetime field caching ([#4558](https://github.com/opensearch-project/OpenSearch/issues/4558))

### Deprecated

Expand Down Expand Up @@ -89,6 +90,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- Async blob read support for encrypted containers ([#10131](https://github.com/opensearch-project/OpenSearch/pull/10131))
- Add capability to restrict async durability mode for remote indexes ([#10189](https://github.com/opensearch-project/OpenSearch/pull/10189))
- Add Doc Status Counter for Indexing Engine ([#4562](https://github.com/opensearch-project/OpenSearch/issues/4562))
- Add unreferenced file cleanup count to merge stats ([#10204](https://github.com/opensearch-project/OpenSearch/pull/10204))
- [Remote Store] Add support to restrict creation & deletion if system repository and mutation of immutable settings of system repository ([#9839](https://github.com/opensearch-project/OpenSearch/pull/9839))
- Improve compressed request handling ([#10261](https://github.com/opensearch-project/OpenSearch/pull/10261))

### Dependencies
- Bump `peter-evans/create-or-update-comment` from 2 to 3 ([#9575](https://github.com/opensearch-project/OpenSearch/pull/9575))
Expand All @@ -102,15 +106,19 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- Bump `tibdex/github-app-token` from 1.5.0 to 2.1.0 ([#10125](https://github.com/opensearch-project/OpenSearch/pull/10125))
- Bump `org.wiremock:wiremock-standalone` from 2.35.0 to 3.1.0 ([#9752](https://github.com/opensearch-project/OpenSearch/pull/9752))
- Bump `com.google.http-client:google-http-client-jackson2` from 1.43.2 to 1.43.3 ([#10126](https://github.com/opensearch-project/OpenSearch/pull/10126))
- Bump `org.xerial.snappy:snappy-java` from 1.1.10.3 to 1.1.10.4 ([#10206](https://github.com/opensearch-project/OpenSearch/pull/10206))
- Bump `org.xerial.snappy:snappy-java` from 1.1.10.3 to 1.1.10.5 ([#10206](https://github.com/opensearch-project/OpenSearch/pull/10206), [#10299](https://github.com/opensearch-project/OpenSearch/pull/10299))
- Bump `com.google.api.grpc:proto-google-common-protos` from 2.10.0 to 2.25.0 ([#10208](https://github.com/opensearch-project/OpenSearch/pull/10208))
- Bump `codecov/codecov-action` from 2 to 3 ([#10209](https://github.com/opensearch-project/OpenSearch/pull/10209))
- Bump `org.bouncycastle:bcpkix-jdk15to18` from 1.75 to 1.76 ([#10219](https://github.com/opensearch-project/OpenSearch/pull/10219))`
- Bump `org.bouncycastle:bcprov-jdk15to18` from 1.75 to 1.76 ([#10219](https://github.com/opensearch-project/OpenSearch/pull/10219))`
- Bump `org.bouncycastle:bcmail-jdk15to18` from 1.75 to 1.76 ([#10219](https://github.com/opensearch-project/OpenSearch/pull/10219))`
- Bump Lucene from 9.7.0 to 9.8.0 ([#10276](https://github.com/opensearch-project/OpenSearch/pull/10276))
- Bump `org.bouncycastle:bcpkix-jdk15to18` from 1.75 to 1.76 ([10219](https://github.com/opensearch-project/OpenSearch/pull/10219))`
- Bump `org.bouncycastle:bcprov-jdk15to18` from 1.75 to 1.76 ([10219](https://github.com/opensearch-project/OpenSearch/pull/10219))`
- Bump `org.bouncycastle:bcmail-jdk15to18` from 1.75 to 1.76 ([10219](https://github.com/opensearch-project/OpenSearch/pull/10219))`
- Bump Lucene from 9.7.0 to 9.8.0 ([10276](https://github.com/opensearch-project/OpenSearch/pull/10276))
- Bump `com.netflix.nebula.ospackage-base` from 11.4.0 to 11.5.0 ([#10295](https://github.com/opensearch-project/OpenSearch/pull/10295))
- Bump asm from 9.5 to 9.6 ([#10302](https://github.com/opensearch-project/OpenSearch/pull/10302))
- Bump netty from 4.1.97.Final to 4.1.99.Final ([#10303](https://github.com/opensearch-project/OpenSearch/pull/10303))
- Bump `peter-evans/create-pull-request` from 3 to 5 ([#10301](https://github.com/opensearch-project/OpenSearch/pull/10301))
- Bump `org.apache.avro:avro` from 1.11.2 to 1.11.3 ([#10210](https://github.com/opensearch-project/OpenSearch/pull/10210))
- Bump `org.bouncycastle:bc-fips` from 1.0.2.3 to 1.0.2.4 ([#10297](https://github.com/opensearch-project/OpenSearch/pull/10297))

### Changed
- Add instrumentation in rest and network layer. ([#9415](https://github.com/opensearch-project/OpenSearch/pull/9415))
Expand All @@ -121,6 +129,11 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- [Tracing Framework] Add support for SpanKind. ([#10122](https://github.com/opensearch-project/OpenSearch/pull/10122))
- Pass parent filter to inner query in nested query ([#10246](https://github.com/opensearch-project/OpenSearch/pull/10246))
- Disable concurrent segment search when terminate_after is used ([#10200](https://github.com/opensearch-project/OpenSearch/pull/10200))
- Add instrumentation in Inbound Handler. ([#100143](https://github.com/opensearch-project/OpenSearch/pull/10143))
- Enable remote segment upload backpressure by default ([#10356](https://github.com/opensearch-project/OpenSearch/pull/10356))
- [Remote Store] Add support to reload repository metadata inplace ([#9569](https://github.com/opensearch-project/OpenSearch/pull/9569))
- [Metrics Framework] Add Metrics framework. ([#10241](https://github.com/opensearch-project/OpenSearch/pull/10241))
- Updating the separator for RemoteStoreLockManager since underscore is allowed in base64UUID url charset ([#10379](https://github.com/opensearch-project/OpenSearch/pull/10379))

### Deprecated

Expand All @@ -132,8 +145,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- Fix broken backward compatibility from 2.7 for IndexSorted field indices ([#10045](https://github.com/opensearch-project/OpenSearch/pull/10045))
- Fix concurrent search NPE when track_total_hits, terminate_after and size=0 are used ([#10082](https://github.com/opensearch-project/OpenSearch/pull/10082))
- Fix remove ingest processor handing ignore_missing parameter not correctly ([10089](https://github.com/opensearch-project/OpenSearch/pull/10089))
- Fix class_cast_exception when passing int to _version and other metadata fields in ingest simulate API ([#10101](https://github.com/opensearch-project/OpenSearch/pull/10101))
- Fix circular dependency in Settings initialization ([10194](https://github.com/opensearch-project/OpenSearch/pull/10194))
- Fix registration and initialization of multiple extensions ([10256](https://github.com/opensearch-project/OpenSearch/pull/10256))
- Fix Segment Replication ShardLockObtainFailedException bug during index corruption ([10370](https://github.com/opensearch-project/OpenSearch/pull/10370))

### Security

Expand Down
2 changes: 1 addition & 1 deletion buildSrc/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ dependencies {
api localGroovy()

api 'commons-codec:commons-codec:1.16.0'
api 'org.apache.commons:commons-compress:1.23.0'
api 'org.apache.commons:commons-compress:1.24.0'
api 'org.apache.ant:ant:1.10.14'
api 'com.netflix.nebula:gradle-extra-configurations-plugin:10.0.0'
api 'com.netflix.nebula:nebula-publishing-plugin:20.3.0'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@ public class TaskInfo {
private long runningTimeNanos;
private boolean cancellable;
private boolean cancelled;
private Long cancellationStartTime;
private TaskId parentTaskId;
private final Map<String, Object> status = new HashMap<>();
private final Map<String, String> headers = new HashMap<>();
Expand Down Expand Up @@ -127,6 +128,14 @@ void setCancelled(boolean cancelled) {
this.cancelled = cancelled;
}

public Long getCancellationStartTime() {
return this.cancellationStartTime;
}

public void setCancellationStartTime(Long cancellationStartTime) {
this.cancellationStartTime = cancellationStartTime;
}

public TaskId getParentTaskId() {
return parentTaskId;
}
Expand Down Expand Up @@ -180,6 +189,7 @@ private void noOpParse(Object s) {}
parser.declareString(TaskInfo::setParentTaskId, new ParseField("parent_task_id"));
parser.declareObject(TaskInfo::setHeaders, (p, c) -> p.mapStrings(), new ParseField("headers"));
parser.declareObject(TaskInfo::setResourceStats, (p, c) -> p.map(), new ParseField("resource_stats"));
parser.declareLong(TaskInfo::setCancellationStartTime, new ParseField("cancellation_time_millis"));
PARSER = (XContentParser p, Void v, String name) -> parser.parse(p, new TaskInfo(new TaskId(name)), null);
}

Expand All @@ -199,7 +209,8 @@ && isCancelled() == taskInfo.isCancelled()
&& Objects.equals(getParentTaskId(), taskInfo.getParentTaskId())
&& Objects.equals(status, taskInfo.status)
&& Objects.equals(getHeaders(), taskInfo.getHeaders())
&& Objects.equals(getResourceStats(), taskInfo.getResourceStats());
&& Objects.equals(getResourceStats(), taskInfo.getResourceStats())
&& Objects.equals(getCancellationStartTime(), taskInfo.cancellationStartTime);
}

@Override
Expand All @@ -216,7 +227,8 @@ public int hashCode() {
getParentTaskId(),
status,
getHeaders(),
getResourceStats()
getResourceStats(),
getCancellationStartTime()
);
}

Expand Down Expand Up @@ -250,6 +262,8 @@ public String toString() {
+ headers
+ ", resource_stats="
+ resourceStats
+ ", cancellationStartTime="
+ cancellationStartTime
+ '}';
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,10 @@ protected CancelTasksResponseTests.ByNodeCancelTasksResponse createServerTestIns
for (int i = 0; i < 4; i++) {
boolean cancellable = randomBoolean();
boolean cancelled = cancellable == true ? randomBoolean() : false;
Long cancellationStartTime = null;
if (cancelled) {
cancellationStartTime = randomNonNegativeLong();
}
tasks.add(
new org.opensearch.tasks.TaskInfo(
new TaskId(NODE_ID, (long) i),
Expand All @@ -97,7 +101,8 @@ protected CancelTasksResponseTests.ByNodeCancelTasksResponse createServerTestIns
cancelled,
new TaskId("node1", randomLong()),
Collections.singletonMap("x-header-of", "some-value"),
null
null,
cancellationStartTime
)
);
}
Expand Down Expand Up @@ -135,6 +140,7 @@ protected void assertInstances(
assertEquals(ti.isCancelled(), taskInfo.isCancelled());
assertEquals(ti.getParentTaskId().getNodeId(), taskInfo.getParentTaskId().getNodeId());
assertEquals(ti.getParentTaskId().getId(), taskInfo.getParentTaskId().getId());
assertEquals(ti.getCancellationStartTime(), taskInfo.getCancellationStartTime());
FakeTaskStatus status = (FakeTaskStatus) ti.getStatus();
assertEquals(status.code, taskInfo.getStatus().get("code"));
assertEquals(status.status, taskInfo.getStatus().get("status"));
Expand Down
2 changes: 1 addition & 1 deletion distribution/packages/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ import java.util.regex.Pattern
*/

plugins {
id "com.netflix.nebula.ospackage-base" version "11.4.0"
id "com.netflix.nebula.ospackage-base" version "11.5.0"
}

void addProcessFilesTask(String type, boolean jdk) {
Expand Down
6 changes: 6 additions & 0 deletions distribution/src/config/opensearch.yml
Original file line number Diff line number Diff line change
Expand Up @@ -121,3 +121,9 @@ ${path.logs}
# index searcher threadpool.
#
#opensearch.experimental.feature.concurrent_segment_search.enabled: false
#
#
# Gates the optimization of datetime formatters caching along with change in default datetime formatter
# Once there is no observed impact on performance, this feature flag can be removed.
#
#opensearch.experimental.optimization.datetime_formatter_caching.enabled: false
2 changes: 1 addition & 1 deletion distribution/tools/plugin-cli/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ dependencies {
compileOnly project(":server")
compileOnly project(":libs:opensearch-cli")
api "org.bouncycastle:bcpg-fips:1.0.7.1"
api "org.bouncycastle:bc-fips:1.0.2.3"
api "org.bouncycastle:bc-fips:1.0.2.4"
testImplementation project(":test:framework")
testImplementation 'com.google.jimfs:jimfs:1.3.0'
testRuntimeOnly("com.google.guava:guava:${versions.guava}") {
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
9008d04fc13da6455e6a792935b93b629757335d
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

package org.opensearch.telemetry.metrics;

import org.opensearch.common.annotation.ExperimentalApi;
import org.opensearch.telemetry.metrics.tags.Tags;

/**
* Counter adds the value to the existing metric.
* {@opensearch.experimental}
*/
@ExperimentalApi
public interface Counter {

/**
* add value.
* @param value value to be added.
*/
void add(double value);

/**
* add value along with the attributes.
*
* @param value value to be added.
* @param tags attributes/dimensions of the metric.
*/
void add(double value, Tags tags);

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

package org.opensearch.telemetry.metrics;

import java.io.IOException;

/**
* Default implementation for {@link MetricsRegistry}
*/
class DefaultMetricsRegistry implements MetricsRegistry {
private final MetricsTelemetry metricsTelemetry;

/**
* Constructor
* @param metricsTelemetry metrics telemetry.
*/
public DefaultMetricsRegistry(MetricsTelemetry metricsTelemetry) {
this.metricsTelemetry = metricsTelemetry;
}

@Override
public Counter createCounter(String name, String description, String unit) {
return metricsTelemetry.createCounter(name, description, unit);
}

@Override
public Counter createUpDownCounter(String name, String description, String unit) {
return metricsTelemetry.createUpDownCounter(name, description, unit);
}

@Override
public void close() throws IOException {
metricsTelemetry.close();
}
}
Loading

0 comments on commit d0d1ccf

Please sign in to comment.