Skip to content

Commit

Permalink
Merge branch 'main' of github.com:opensearch-project/OpenSearch into …
Browse files Browse the repository at this point in the history
…keyword-iterator
  • Loading branch information
bharath-techie committed Oct 11, 2024
2 parents 3571ec8 + 1e7f6df commit a48de57
Show file tree
Hide file tree
Showing 92 changed files with 1,540 additions and 370 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/delete_backport_branch.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ jobs:
runs-on: ubuntu-latest
permissions:
contents: write
if: startsWith(github.event.pull_request.head.ref,'backport/')
if: github.repository == 'opensearch-project/OpenSearch' && startsWith(github.event.pull_request.head.ref,'backport/')
steps:
- name: Delete merged branch
uses: actions/github-script@v7
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/precommit.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ jobs:
runs-on: ${{ matrix.os }}
strategy:
matrix:
java: [ 11, 17, 21 ]
java: [ 11, 17, 21, 23 ]
os: [ubuntu-latest, windows-latest, macos-latest, macos-13]
steps:
- uses: actions/checkout@v4
Expand Down
7 changes: 7 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
### Added
- Add support for async deletion in S3BlobContainer ([#15621](https://github.com/opensearch-project/OpenSearch/pull/15621))
- MultiTermQueries in keyword fields now default to `indexed` approach and gated behind cluster setting ([#15637](https://github.com/opensearch-project/OpenSearch/pull/15637))
- [Workload Management] Add Integration Tests for Workload Management CRUD APIs ([#15955](https://github.com/opensearch-project/OpenSearch/pull/15955))
- [Workload Management] QueryGroup resource cancellation framework changes ([#15651](https://github.com/opensearch-project/OpenSearch/pull/15651))
- [Workload Management] Add orchestrator for wlm resiliency (QueryGroupService) ([#15925](https://github.com/opensearch-project/OpenSearch/pull/15925))
- [Workload Management] Add QueryGroup Stats API Logic ([15777](https://github.com/opensearch-project/OpenSearch/pull/15777))
- Fallback to Remote cluster-state on Term-Version check mismatch - ([#15424](https://github.com/opensearch-project/OpenSearch/pull/15424))
- Implement WithFieldName interface in ValuesSourceAggregationBuilder & FieldSortBuilder ([#15916](https://github.com/opensearch-project/OpenSearch/pull/15916))
- Add successfulSearchShardIndices in searchRequestContext ([#15967](https://github.com/opensearch-project/OpenSearch/pull/15967), [#16110](https://github.com/opensearch-project/OpenSearch/pull/16110))
Expand Down Expand Up @@ -42,6 +44,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- Bump `com.azure:azure-core-http-netty` from 1.15.3 to 1.15.4 ([#16133](https://github.com/opensearch-project/OpenSearch/pull/16133))
- Bump `org.jline:jline` from 3.26.3 to 3.27.0 ([#16135](https://github.com/opensearch-project/OpenSearch/pull/16135))
- Bump `netty` from 4.1.112.Final to 4.1.114.Final ([#16182](https://github.com/opensearch-project/OpenSearch/pull/16182))
- Bump `com.google.api-client:google-api-client` from 2.2.0 to 2.7.0 ([#16216](https://github.com/opensearch-project/OpenSearch/pull/16216))
- Bump `com.azure:azure-json` from 1.1.0 to 1.3.0 ([#16217](https://github.com/opensearch-project/OpenSearch/pull/16217))
- Bump `io.grpc:grpc-api` from 1.57.2 to 1.68.0 ([#16213](https://github.com/opensearch-project/OpenSearch/pull/16213))
- Bump `com.squareup.okio:okio` from 3.9.0 to 3.9.1 ([#16212](https://github.com/opensearch-project/OpenSearch/pull/16212))

### Changed
- Add support for docker compose v2 in TestFixturesPlugin ([#16049](https://github.com/opensearch-project/OpenSearch/pull/16049))
Expand All @@ -64,6 +70,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- Fix race condition in node-join and node-left ([#15521](https://github.com/opensearch-project/OpenSearch/pull/15521))
- Streaming bulk request hangs ([#16158](https://github.com/opensearch-project/OpenSearch/pull/16158))
- Fix warnings from SLF4J on startup when repository-s3 is installed ([#16194](https://github.com/opensearch-project/OpenSearch/pull/16194))
- Fix protobuf-java leak through client library dependencies ([#16254](https://github.com/opensearch-project/OpenSearch/pull/16254))

### Security

Expand Down
2 changes: 1 addition & 1 deletion buildSrc/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ dependencies {
api 'com.github.johnrengelman:shadow:8.1.1'
api 'org.jdom:jdom2:2.0.6.1'
api "org.jetbrains.kotlin:kotlin-stdlib-jdk8:${props.getProperty('kotlin')}"
api 'de.thetaphi:forbiddenapis:3.6'
api 'de.thetaphi:forbiddenapis:3.8'
api 'com.avast.gradle:gradle-docker-compose-plugin:0.17.6'
api "org.yaml:snakeyaml:${props.getProperty('snakeyaml')}"
api 'org.apache.maven:maven-model:3.9.6'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ public class ThirdPartyAuditPrecommitPlugin extends PrecommitPlugin {
public TaskProvider<? extends Task> createTask(Project project) {
project.getPlugins().apply(CompileOnlyResolvePlugin.class);
project.getConfigurations().create("forbiddenApisCliJar");
project.getDependencies().add("forbiddenApisCliJar", "de.thetaphi:forbiddenapis:3.5.1");
project.getDependencies().add("forbiddenApisCliJar", "de.thetaphi:forbiddenapis:3.8");

Configuration jdkJarHellConfig = project.getConfigurations().create(JDK_JAR_HELL_CONFIG_NAME);
if (BuildParams.isInternal() && project.getPath().equals(":libs:opensearch-core") == false) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -77,9 +77,9 @@
import java.util.stream.Stream;

public class DistroTestPlugin implements Plugin<Project> {
private static final String SYSTEM_JDK_VERSION = "21.0.4+7";
private static final String SYSTEM_JDK_VERSION = "23+37";
private static final String SYSTEM_JDK_VENDOR = "adoptium";
private static final String GRADLE_JDK_VERSION = "21.0.4+7";
private static final String GRADLE_JDK_VERSION = "23+37";
private static final String GRADLE_JDK_VENDOR = "adoptium";

// all distributions used by distro tests. this is temporary until tests are per distribution
Expand Down
7 changes: 4 additions & 3 deletions buildSrc/version.properties
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ opensearch = 3.0.0
lucene = 9.12.0

bundled_jdk_vendor = adoptium
bundled_jdk = 21.0.4+7
bundled_jdk = 23+37

# optional dependencies
spatial4j = 0.7
Expand All @@ -25,6 +25,7 @@ jakarta_annotation = 1.3.5
google_http_client = 1.44.1
tdigest = 3.3
hdrhistogram = 2.2.2
grpc = 1.68.0

# when updating the JNA version, also update the version in buildSrc/build.gradle
jna = 5.13.0
Expand Down Expand Up @@ -59,9 +60,9 @@ bouncycastle=1.78
randomizedrunner = 2.7.1
junit = 4.13.2
hamcrest = 2.1
mockito = 5.12.0
mockito = 5.14.1
objenesis = 3.2
bytebuddy = 1.14.9
bytebuddy = 1.15.4

# benchmark dependencies
jmh = 1.35
Expand Down
4 changes: 3 additions & 1 deletion client/rest-high-level/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,9 @@ restResources {
}

dependencies {
api project(':server')
api(project(':server')) {
exclude group: 'com.google.protobuf'
}
api project(':client:rest')
api project(':modules:mapper-extras')
api project(':modules:parent-join')
Expand Down
4 changes: 1 addition & 3 deletions gradle.properties
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,7 @@ systemProp.org.gradle.dependency.duplicate.project.detection=false
# Enforce the build to fail on deprecated gradle api usage
systemProp.org.gradle.warning.mode=fail

# forcing to use TLS1.2 to avoid failure in vault
# see https://github.com/hashicorp/vault/issues/8750#issuecomment-631236121
systemProp.jdk.tls.client.protocols=TLSv1.2
systemProp.jdk.tls.client.protocols=TLSv1.2,TLSv1.3

# jvm args for faster test execution by default
systemProp.tests.jvm.argline=-XX:TieredStopAtLevel=1 -XX:ReservedCodeCacheSize=64m
2 changes: 1 addition & 1 deletion gradle/code-coverage.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ repositories {

allprojects {
plugins.withId('jacoco') {
jacoco.toolVersion = '0.8.10'
jacoco.toolVersion = '0.8.12'
}
}

Expand Down
2 changes: 1 addition & 1 deletion libs/core/src/main/java/org/opensearch/Version.java
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ public class Version implements Comparable<Version>, ToXContentFragment {
public static final Version V_2_17_0 = new Version(2170099, org.apache.lucene.util.Version.LUCENE_9_11_1);
public static final Version V_2_17_1 = new Version(2170199, org.apache.lucene.util.Version.LUCENE_9_11_1);
public static final Version V_2_17_2 = new Version(2170299, org.apache.lucene.util.Version.LUCENE_9_11_1);
public static final Version V_2_18_0 = new Version(2180099, org.apache.lucene.util.Version.LUCENE_9_11_1);
public static final Version V_2_18_0 = new Version(2180099, org.apache.lucene.util.Version.LUCENE_9_12_0);
public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_12_0);
public static final Version CURRENT = V_3_0_0;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -114,8 +114,8 @@ public GrokCaptureExtracter forBoolean(Function<Consumer<Boolean>, GrokCaptureEx
/**
* Build an extract that has access to the "native" type of the extracter
* match. This means that patterns like {@code %{NUMBER:bytes:float}} has
* access to an actual {@link float}. Extracters returned from this method
* should be stateless stateless and can be reused. Pathological implementations
* access to an actual float. Extracters returned from this method
* should be stateless and can be reused. Pathological implementations
* of the {@code map} parameter could violate this, but the caller should
* take care to stay sane.
* <p>
Expand Down Expand Up @@ -144,27 +144,27 @@ public interface NativeExtracterMap<T> {
T forString(Function<Consumer<String>, GrokCaptureExtracter> buildExtracter);

/**
* Called when the native type is an {@link int}.
* Called when the native type is an int.
*/
T forInt(Function<IntConsumer, GrokCaptureExtracter> buildExtracter);

/**
* Called when the native type is an {@link long}.
* Called when the native type is an long.
*/
T forLong(Function<LongConsumer, GrokCaptureExtracter> buildExtracter);

/**
* Called when the native type is an {@link float}.
* Called when the native type is an float.
*/
T forFloat(Function<FloatConsumer, GrokCaptureExtracter> buildExtracter);

/**
* Called when the native type is an {@link double}.
* Called when the native type is an double.
*/
T forDouble(Function<DoubleConsumer, GrokCaptureExtracter> buildExtracter);

/**
* Called when the native type is an {@link boolean}.
* Called when the native type is an boolean.
*/
T forBoolean(Function<Consumer<Boolean>, GrokCaptureExtracter> buildExtracter);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,7 @@ protected List<TransportAddress> buildDynamicHosts(Settings nodeSettings, int no
exchange.getResponseHeaders().set("Content-Type", "text/xml; charset=UTF-8");
exchange.sendResponseHeaders(HttpStatus.SC_OK, responseBody.length);
exchange.getResponseBody().write(responseBody);
exchange.getResponseBody().flush();
return;
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,7 @@ public void testEC2DiscoveryRetriesOnRateLimiting() throws IOException {
exchange.getResponseHeaders().set("Content-Type", "text/xml; charset=UTF-8");
exchange.sendResponseHeaders(HttpStatus.SC_OK, responseBody.length);
exchange.getResponseBody().write(responseBody);
exchange.getResponseBody().flush();
return;
}
}
Expand Down
2 changes: 1 addition & 1 deletion plugins/discovery-gce/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ dependencies {
api "commons-logging:commons-logging:${versions.commonslogging}"
api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
api "commons-codec:commons-codec:${versions.commonscodec}"
api 'io.grpc:grpc-api:1.57.2'
api "io.grpc:grpc-api:${versions.grpc}"
api 'io.opencensus:opencensus-api:0.31.1'
api 'io.opencensus:opencensus-contrib-http-util:0.31.1'
runtimeOnly "com.google.guava:guava:${versions.guava}"
Expand Down
1 change: 0 additions & 1 deletion plugins/discovery-gce/licenses/grpc-api-1.57.2.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions plugins/discovery-gce/licenses/grpc-api-1.68.0.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
9a9f25c58d8d5b0fcf37ae889a50fec87e34ac08
2 changes: 1 addition & 1 deletion plugins/repository-azure/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ opensearchplugin {

dependencies {
api 'com.azure:azure-core:1.51.0'
api 'com.azure:azure-json:1.1.0'
api 'com.azure:azure-json:1.3.0'
api 'com.azure:azure-xml:1.1.0'
api 'com.azure:azure-storage-common:12.25.1'
api 'com.azure:azure-core-http-netty:1.15.4'
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
11b6a0708e9d6c90a1a76574c7720edce47dacc1
4 changes: 2 additions & 2 deletions plugins/repository-gcs/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ dependencies {

api 'com.google.apis:google-api-services-storage:v1-rev20230617-2.0.0'

api 'com.google.api-client:google-api-client:2.2.0'
api 'com.google.api-client:google-api-client:2.7.0'

api 'com.google.api.grpc:proto-google-common-protos:2.37.1'
api 'com.google.api.grpc:proto-google-iam-v1:1.33.0'
Expand Down Expand Up @@ -86,7 +86,7 @@ dependencies {
api "org.apache.logging.log4j:log4j-1.2-api:${versions.log4j}"
api "commons-codec:commons-codec:${versions.commonscodec}"
api 'org.threeten:threetenbp:1.4.4'
api 'io.grpc:grpc-api:1.57.2'
api "io.grpc:grpc-api:${versions.grpc}"
api 'io.opencensus:opencensus-api:0.31.1'
api 'io.opencensus:opencensus-contrib-http-util:0.31.1'

Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
59c8e5e3c03f146561a83051af3ca945d40e02c6
1 change: 0 additions & 1 deletion plugins/repository-gcs/licenses/grpc-api-1.57.2.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions plugins/repository-gcs/licenses/grpc-api-1.68.0.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
9a9f25c58d8d5b0fcf37ae889a50fec87e34ac08
4 changes: 4 additions & 0 deletions plugins/repository-hdfs/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,10 @@ for (String fixtureName : ['hdfsFixture', 'haHdfsFixture', 'secureHdfsFixture',
}
final List<String> miniHDFSArgs = []

if (BuildParams.runtimeJavaVersion >= JavaVersion.VERSION_23) {
miniHDFSArgs.add('-Djava.security.manager=allow')
}

// If it's a secure fixture, then depend on Kerberos Fixture and principals + add the krb5conf to the JVM options
if (fixtureName.equals('secureHdfsFixture') || fixtureName.equals('secureHaHdfsFixture')) {
miniHDFSArgs.add("-Djava.security.krb5.conf=${project(':test:fixtures:krb5kdc-fixture').ext.krb5Conf("hdfs")}");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,7 @@ private static int allocatedProcessors(Settings settings) {
}

private static int urgentPoolCount(Settings settings) {
return boundedBy((allocatedProcessors(settings) + 7) / 8, 1, 2);
return boundedBy((allocatedProcessors(settings) + 1) / 2, 1, 2);
}

private static int priorityPoolCount(Settings settings) {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

package org.opensearch.repositories.s3;

import org.opensearch.common.settings.Settings;
import org.opensearch.common.unit.SizeUnit;
import org.opensearch.common.unit.SizeValue;
import org.opensearch.common.util.concurrent.OpenSearchThreadPoolExecutor;
import org.opensearch.test.OpenSearchTestCase;
import org.opensearch.threadpool.ExecutorBuilder;
import org.opensearch.threadpool.ThreadPool;
import org.opensearch.threadpool.ThreadPool.ThreadPoolType;

import java.io.IOException;
import java.nio.file.Path;
import java.util.List;
import java.util.concurrent.Executor;

import static org.hamcrest.CoreMatchers.instanceOf;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.notNullValue;

public class S3RepositoryPluginTests extends OpenSearchTestCase {

private static final String URGENT_FUTURE_COMPLETION = "urgent_future_completion";

public void testGetExecutorBuilders() throws IOException {
final int processors = randomIntBetween(1, 64);
Settings settings = Settings.builder().put("node.name", "test").put("node.processors", processors).build();
Path configPath = createTempDir();
ThreadPool threadPool = null;
try (S3RepositoryPlugin plugin = new S3RepositoryPlugin(settings, configPath)) {
List<ExecutorBuilder<?>> executorBuilders = plugin.getExecutorBuilders(settings);
assertNotNull(executorBuilders);
assertFalse(executorBuilders.isEmpty());
threadPool = new ThreadPool(settings, executorBuilders.toArray(new ExecutorBuilder<?>[0]));
final Executor executor = threadPool.executor(URGENT_FUTURE_COMPLETION);
assertNotNull(executor);
assertThat(executor, instanceOf(OpenSearchThreadPoolExecutor.class));
final OpenSearchThreadPoolExecutor openSearchThreadPoolExecutor = (OpenSearchThreadPoolExecutor) executor;
final ThreadPool.Info info = threadPool.info(URGENT_FUTURE_COMPLETION);
int size = boundedBy((processors + 1) / 2, 1, 2);
assertThat(info.getName(), equalTo(URGENT_FUTURE_COMPLETION));
assertThat(info.getThreadPoolType(), equalTo(ThreadPoolType.FIXED));
assertThat(info.getQueueSize(), notNullValue());
assertThat(info.getQueueSize(), equalTo(new SizeValue(10, SizeUnit.KILO)));
assertThat(openSearchThreadPoolExecutor.getQueue().remainingCapacity(), equalTo(10_000));

assertThat(info.getMin(), equalTo(size));
assertThat(openSearchThreadPoolExecutor.getCorePoolSize(), equalTo(size));
assertThat(info.getMax(), equalTo(size));
assertThat(openSearchThreadPoolExecutor.getMaximumPoolSize(), equalTo(size));

final int availableProcessors = Runtime.getRuntime().availableProcessors();
if (processors > availableProcessors) {
assertWarnings(
"setting [node.processors] to value ["
+ processors
+ "] which is more than available processors ["
+ availableProcessors
+ "] is deprecated"
);
}
} finally {
if (threadPool != null) {
terminate(threadPool);
}
}
}

private static int boundedBy(int value, int min, int max) {
return Math.min(max, Math.max(min, value));
}

}
1 change: 1 addition & 0 deletions plugins/workload-management/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
*/

apply plugin: 'opensearch.yaml-rest-test'
apply plugin: 'opensearch.java-rest-test'
apply plugin: 'opensearch.internal-cluster-test'

opensearchplugin {
Expand Down
Loading

0 comments on commit a48de57

Please sign in to comment.