Skip to content

Commit

Permalink
Merge remote-tracking branch 'upstream/main'
Browse files Browse the repository at this point in the history
  • Loading branch information
Bukhtawar committed Aug 19, 2024
2 parents 35e804a + 096c354 commit 84f8094
Show file tree
Hide file tree
Showing 150 changed files with 5,143 additions and 358 deletions.
3 changes: 2 additions & 1 deletion .ci/java-versions.properties
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@
# build and test OpenSearch for this branch. Valid Java versions
# are 'java' or 'openjdk' followed by the major release number.

OPENSEARCH_BUILD_JAVA=openjdk11
# See please https://docs.gradle.org/8.10/userguide/upgrading_version_8.html#minimum_daemon_jvm_version
OPENSEARCH_BUILD_JAVA=openjdk17
OPENSEARCH_RUNTIME_JAVA=java11
GRADLE_TASK=build
GRADLE_EXTRA_ARGS=
16 changes: 14 additions & 2 deletions .github/workflows/assemble.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,17 @@ jobs:
with:
java-version: ${{ matrix.java }}
distribution: temurin
- name: Set up JDK 17
# See please https://docs.gradle.org/8.10/userguide/upgrading_version_8.html#minimum_daemon_jvm_version
if: matrix.java == 11
uses: actions/setup-java@v4
with:
java-version: 17
distribution: temurin
- name: Set JAVA${{ matrix.java }}_HOME
shell: bash
run: |
echo "JAVA${{ matrix.java }}_HOME=$JAVA_HOME_${{ matrix.java }}_${{ runner.arch }}" >> $GITHUB_ENV
- name: Setup docker (missing on MacOS)
id: setup_docker
if: runner.os == 'macos'
Expand All @@ -30,10 +41,11 @@ jobs:
# Report success even if previous step failed (Docker on MacOS runner is very unstable)
exit 0;
- name: Run Gradle (assemble)
shell: bash
if: runner.os != 'macos'
run: |
./gradlew assemble --parallel --no-build-cache -PDISABLE_BUILD_CACHE
./gradlew assemble --parallel --no-build-cache -PDISABLE_BUILD_CACHE -Druntime.java=${{ matrix.java }}
- name: Run Gradle (assemble)
if: runner.os == 'macos' && steps.setup_docker.outcome == 'success'
run: |
./gradlew assemble --parallel --no-build-cache -PDISABLE_BUILD_CACHE
./gradlew assemble --parallel --no-build-cache -PDISABLE_BUILD_CACHE -Druntime.java=${{ matrix.java }}
20 changes: 16 additions & 4 deletions .github/workflows/benchmark-pull-request.yml
Original file line number Diff line number Diff line change
Expand Up @@ -108,13 +108,25 @@ jobs:
echo "prHeadRepo=$headRepo" >> $GITHUB_ENV
echo "prHeadRefSha=$headRefSha" >> $GITHUB_ENV
- id: get_approvers
run: |
echo "approvers=$(cat .github/CODEOWNERS | grep '^\*' | tr -d '* ' | sed 's/@/,/g' | sed 's/,//1')" >> $GITHUB_OUTPUT
uses: actions/github-script@v7
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
result-encoding: string
script: |
// Get the collaborators - filtered to maintainer permissions
const maintainersResponse = await github.request('GET /repos/{owner}/{repo}/collaborators', {
owner: context.repo.owner,
repo: context.repo.repo,
permission: 'maintain',
affiliation: 'all',
per_page: 100
});
return maintainersResponse.data.map(item => item.login).join(', ');
- uses: trstringer/manual-approval@v1
if: (!contains(steps.get_approvers.outputs.approvers, github.event.comment.user.login))
if: (!contains(steps.get_approvers.outputs.result, github.event.comment.user.login))
with:
secret: ${{ github.TOKEN }}
approvers: ${{ steps.get_approvers.outputs.approvers }}
approvers: ${{ steps.get_approvers.outputs.result }}
minimum-approvals: 1
issue-title: 'Request to approve/deny benchmark run for PR #${{ env.PR_NUMBER }}'
issue-body: "Please approve or deny the benchmark run for PR #${{ env.PR_NUMBER }}"
Expand Down
17 changes: 14 additions & 3 deletions .github/workflows/changelog_verifier.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,18 +27,29 @@ jobs:
continue-on-error: true
- run: |
# The check was possibly skipped leading to success for both the jobs
has_backport_label=${{ contains(join(github.event.pull_request.labels.*.name, ', '), 'backport')}}
has_breaking_label=${{ contains(join(github.event.pull_request.labels.*.name, ', '), '>breaking')}}
if [[ $has_breaking_label == true && $has_backport_label == true ]]; then
echo "error: Please make sure that the PR does not have a backport label associated with it when making breaking changes"
exit 1
fi
if [[ ${{ steps.verify-changelog-3x.outcome }} == 'success' && ${{ steps.verify-changelog.outcome }} == 'success' ]]; then
exit 0
fi
if [[ ${{ steps.verify-changelog-3x.outcome }} == 'failure' && ${{ steps.verify-changelog.outcome }} == 'failure' ]]; then
echo "error: Please ensure a changelog entry exists in CHANGELOG.md or CHANGELOG-3.0.md"
exit 1
fi
# Concatenates the labels and checks if the string contains "backport"
has_backport_label=${{ contains(join(github.event.pull_request.labels.*.name, ', '), 'backport')}}
if [[ ${{ steps.verify-changelog.outcome }} == 'success' && $has_backport_label == false ]]; then
echo "error: Please make sure that the PR has a backport label associated with it when making an entry to the CHANGELOG.md file"
exit 1
fi
if [[ ${{ steps.verify-changelog-3x.outcome }} == 'success' && $has_backport_label == true ]]; then
echo "error: Please make sure that the PR does not have a backport label associated with it when making an entry to the CHANGELOG-3.0.md file"
exit 1
fi
7 changes: 7 additions & 0 deletions .github/workflows/dependabot_pr.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,13 @@ jobs:
with:
token: ${{ steps.github_app_token.outputs.token }}

# See please https://docs.gradle.org/8.10/userguide/upgrading_version_8.html#minimum_daemon_jvm_version
- name: Set up JDK 17
uses: actions/setup-java@v4
with:
java-version: 17
distribution: temurin

- name: Update Gradle SHAs
run: |
./gradlew updateSHAs
Expand Down
14 changes: 13 additions & 1 deletion .github/workflows/precommit.yml
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,18 @@ jobs:
java-version: ${{ matrix.java }}
distribution: temurin
cache: gradle
- name: Set up JDK 17
# See please https://docs.gradle.org/8.10/userguide/upgrading_version_8.html#minimum_daemon_jvm_version
if: matrix.java == 11
uses: actions/setup-java@v4
with:
java-version: 17
distribution: temurin
- name: Set JAVA${{ matrix.java }}_HOME
shell: bash
run: |
echo "JAVA${{ matrix.java }}_HOME=$JAVA_HOME_${{ matrix.java }}_${{ runner.arch }}" >> $GITHUB_ENV
- name: Run Gradle (precommit)
shell: bash
run: |
./gradlew javadoc precommit --parallel
./gradlew javadoc precommit --parallel -Druntime.java=${{ matrix.java }}
10 changes: 10 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,16 +5,19 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),

## [Unreleased 2.x]
### Added
- [Offline Nodes] Adds offline-tasks library containing various interfaces to be used for Offline Background Tasks. ([#13574](https://github.com/opensearch-project/OpenSearch/pull/13574))
- Fix for hasInitiatedFetching to fix allocation explain and manual reroute APIs (([#14972](https://github.com/opensearch-project/OpenSearch/pull/14972))
- [Workload Management] Add queryGroupId to Task ([14708](https://github.com/opensearch-project/OpenSearch/pull/14708))
- Add setting to ignore throttling nodes for allocation of unassigned primaries in remote restore ([#14991](https://github.com/opensearch-project/OpenSearch/pull/14991))
- [Streaming Indexing] Enhance RestClient with a new streaming API support ([#14437](https://github.com/opensearch-project/OpenSearch/pull/14437))
- Add basic aggregation support for derived fields ([#14618](https://github.com/opensearch-project/OpenSearch/pull/14618))
- [Workload Management] Add Create QueryGroup API Logic ([#14680](https://github.com/opensearch-project/OpenSearch/pull/14680))- [Workload Management] Add Create QueryGroup API Logic ([#14680](https://github.com/opensearch-project/OpenSearch/pull/14680))
- Add ThreadContextPermission for markAsSystemContext and allow core to perform the method ([#15016](https://github.com/opensearch-project/OpenSearch/pull/15016))
- Add ThreadContextPermission for stashAndMergeHeaders and stashWithOrigin ([#15039](https://github.com/opensearch-project/OpenSearch/pull/15039))
- [Concurrent Segment Search] Support composite aggregations with scripting ([#15072](https://github.com/opensearch-project/OpenSearch/pull/15072))
- Add `rangeQuery` and `regexpQuery` for `constant_keyword` field type ([#14711](https://github.com/opensearch-project/OpenSearch/pull/14711))
- Add took time to request nodes stats ([#15054](https://github.com/opensearch-project/OpenSearch/pull/15054))
- [Workload Management] Add Get QueryGroup API Logic ([14709](https://github.com/opensearch-project/OpenSearch/pull/14709))
- [Workload Management] QueryGroup resource tracking framework changes ([#13897](https://github.com/opensearch-project/OpenSearch/pull/13897))
- Add slice execution listeners to SearchOperationListener interface ([#15153](https://github.com/opensearch-project/OpenSearch/pull/15153))

Expand All @@ -31,9 +34,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- Bump `com.azure:azure-core` from 1.49.1 to 1.51.0 ([#15111](https://github.com/opensearch-project/OpenSearch/pull/15111))
- Bump `org.xerial.snappy:snappy-java` from 1.1.10.5 to 1.1.10.6 ([#15207](https://github.com/opensearch-project/OpenSearch/pull/15207))
- Bump `com.azure:azure-xml` from 1.0.0 to 1.1.0 ([#15206](https://github.com/opensearch-project/OpenSearch/pull/15206))
- Bump `reactor` from 3.5.19 to 3.5.20 ([#15262](https://github.com/opensearch-project/OpenSearch/pull/15262))
- Bump `reactor-netty` from 1.1.21 to 1.1.22 ([#15262](https://github.com/opensearch-project/OpenSearch/pull/15262))
- Bump `org.apache.kerby:kerb-admin` from 2.0.3 to 2.1.0 ([#15301](https://github.com/opensearch-project/OpenSearch/pull/15301))

### Changed
- Add lower limit for primary and replica batch allocators timeout ([#14979](https://github.com/opensearch-project/OpenSearch/pull/14979))
- Replace and block usages of org.apache.logging.log4j.util.Strings ([#15238](https://github.com/opensearch-project/OpenSearch/pull/15238))

### Deprecated

Expand All @@ -45,6 +52,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
- Fix missing value of FieldSort for unsigned_long ([#14963](https://github.com/opensearch-project/OpenSearch/pull/14963))
- Fix delete index template failed when the index template matches a data stream but is unused ([#15080](https://github.com/opensearch-project/OpenSearch/pull/15080))
- Fix array_index_out_of_bounds_exception when indexing documents with field name containing only dot ([#15126](https://github.com/opensearch-project/OpenSearch/pull/15126))
- Fixed array field name omission in flat_object function for nested JSON ([#13620](https://github.com/opensearch-project/OpenSearch/pull/13620))
- Fix range aggregation optimization ignoring top level queries ([#15194](https://github.com/opensearch-project/OpenSearch/pull/15194))
- Fix incorrect parameter names in MinHash token filter configuration handling ([#15233](https://github.com/opensearch-project/OpenSearch/pull/15233))

### Security

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,9 @@
java.nio.file.Paths @ Use org.opensearch.common.io.PathUtils.get() instead.
java.nio.file.FileSystems#getDefault() @ use org.opensearch.common.io.PathUtils.getDefaultFileSystem() instead.

joptsimple.internal.Strings @ use org.opensearch.core.common.Strings instead.
org.apache.logging.log4j.util.Strings @ use org.opensearch.core.common.Strings instead.

java.nio.file.Files#getFileStore(java.nio.file.Path) @ Use org.opensearch.env.Environment.getFileStore() instead, impacted by JDK-8034057
java.nio.file.Files#isWritable(java.nio.file.Path) @ Use org.opensearch.env.Environment.isWritable() instead, impacted by JDK-8034057

Expand Down
2 changes: 1 addition & 1 deletion buildSrc/src/main/resources/minimumCompilerVersion
Original file line number Diff line number Diff line change
@@ -1 +1 @@
14
17
4 changes: 2 additions & 2 deletions buildSrc/version.properties
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,8 @@ netty = 4.1.112.Final
joda = 2.12.7

# project reactor
reactor_netty = 1.1.21
reactor = 3.5.19
reactor_netty = 1.1.22
reactor = 3.5.20

# client dependencies
httpclient5 = 5.2.3
Expand Down
1 change: 0 additions & 1 deletion client/rest/licenses/reactor-core-3.5.19.jar.sha1

This file was deleted.

1 change: 1 addition & 0 deletions client/rest/licenses/reactor-core-3.5.20.jar.sha1
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
1fc0f91e2b93778a974339d2c24363d7f34f90b4
Binary file modified gradle/wrapper/gradle-wrapper.jar
Binary file not shown.
4 changes: 2 additions & 2 deletions gradle/wrapper/gradle-wrapper.properties
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@

distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-8.9-all.zip
distributionUrl=https\://services.gradle.org/distributions/gradle-8.10-all.zip
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
distributionSha256Sum=258e722ec21e955201e31447b0aed14201765a3bfbae296a46cf60b70e66db70
distributionSha256Sum=682b4df7fe5accdca84a4d1ef6a3a6ab096b3efd5edf7de2bd8c758d95a93703
Original file line number Diff line number Diff line change
Expand Up @@ -375,7 +375,7 @@ private static void skipToListStart(XContentParser parser) throws IOException {
}
}

// read a list without bounds checks, assuming the the current parser is always on an array start
// read a list without bounds checks, assuming the current parser is always on an array start
private static List<Object> readListUnsafe(XContentParser parser, Supplier<Map<String, Object>> mapFactory) throws IOException {
assert parser.currentToken() == Token.START_ARRAY;
ArrayList<Object> list = new ArrayList<>();
Expand Down
25 changes: 25 additions & 0 deletions libs/task-commons/build.gradle
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*
* Modifications Copyright OpenSearch Contributors. See
* GitHub history for details.
*/

dependencies {
api project(':libs:opensearch-common')

testImplementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
testImplementation "junit:junit:${versions.junit}"
testImplementation "org.hamcrest:hamcrest:${versions.hamcrest}"
testImplementation(project(":test:framework")) {
exclude group: 'org.opensearch', module: 'opensearch-task-commons'
}
}

tasks.named('forbiddenApisMain').configure {
replaceSignatureFiles 'jdk-signatures'
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
/*
* SPDX-License-Identifier: Apache-2.0
*
* The OpenSearch Contributors require contributions made to
* this file be licensed under the Apache-2.0 license or a
* compatible open source license.
*/

package org.opensearch.task.commons.clients;

import org.opensearch.task.commons.task.TaskStatus;
import org.opensearch.task.commons.task.TaskType;
import org.opensearch.task.commons.worker.WorkerNode;

/**
* Request object for listing tasks
*/
public class TaskListRequest {

/**
* Filters listTasks response by specific task status'
*/
private TaskStatus[] taskStatus;

/**
* Filter listTasks response by specific task types
*/
private TaskType[] taskTypes;

/**
* Filter listTasks response by specific worker node
*/
private WorkerNode workerNodes;

/**
* Depicts the start page number for the list call.
*
* @see TaskManagerClient#listTasks(TaskListRequest)
*/
private int startPageNumber;

/**
* Depicts the page size for the list call.
*
* @see TaskManagerClient#listTasks(TaskListRequest)
*/
private int pageSize;

/**
* Default constructor
*/
public TaskListRequest() {}

/**
* Update task types to filter with in the request
* @param taskTypes TaskType[]
* @return ListTaskRequest
*/
public TaskListRequest taskType(TaskType... taskTypes) {
this.taskTypes = taskTypes;
return this;
}

/**
* Update task status to filter with in the request
* @param taskStatus TaskStatus[]
* @return ListTaskRequest
*/
public TaskListRequest taskType(TaskStatus... taskStatus) {
this.taskStatus = taskStatus;
return this;
}

/**
* Update worker node to filter with in the request
* @param workerNode WorkerNode
* @return ListTaskRequest
*/
private TaskListRequest workerNode(WorkerNode workerNode) {
this.workerNodes = workerNode;
return this;
}

/**
* Update page number to start with when fetching the list of tasks
* @param startPageNumber startPageNumber
* @return ListTaskRequest
*/
public TaskListRequest startPageNumber(int startPageNumber) {
this.startPageNumber = startPageNumber;
return this;
}

/**
* Update page size for the list tasks response
* @param pageSize int
* @return ListTaskRequest
*/
public TaskListRequest pageSize(int pageSize) {
this.pageSize = pageSize;
return this;
}
}
Loading

0 comments on commit 84f8094

Please sign in to comment.