Merge remote-tracking branch 'refs/remotes/origin/main'

opensearch-project · Sep 29, 2024 · f847731 · f847731
2 parents 9620b94 + 7d05664
commit f847731
Show file tree

Hide file tree

Showing 192 changed files with 8,317 additions and 1,232 deletions.
diff --git a/.cspell b/.cspell
@@ -194,4 +194,6 @@ urldecode
 vectory
 whoamiprotected
 wordnet
-Yrtsd
+Yrtsd
+reprovision
+deprovision
diff --git a/.github/workflows/test-spec.yml b/.github/workflows/test-spec.yml
@@ -30,14 +30,19 @@ jobs:
           - version: 2.16.0
           - version: 2.16.0
             tests: plugins/index_state_management
+          - version: 2.16.0
+            tests: plugins/ml
+          - version: 2.16.0
+            tests: routing
           - version: 2.16.0
             tests: snapshot
           - version: 2.17.0
+          - version: 2.18.0
             hub: opensearchstaging
-            ref: '@sha256:ed4274522a50228f41b50f1a7ea86e6b52fa6737072fc151b2624d22aff80d56'
+            ref: '@sha256:4445e195c53992038891519dc3be0d273cdaad1b047943d68921168ed243e7e9'
           - version: 3.0.0
             hub: opensearchstaging
-            ref: '@sha256:cab6f71b284485c44306f8f4849ad520283c2a32ece617109b38183ba29cc401'
+            ref: '@sha256:cf07c0ffa7d9e8a3e7fdb58041caae3bb81f1123260431b99d0ebf4a52c3d9a3'
 
     name: test-opensearch-spec (version=${{ matrix.entry.version }}, hub=${{ matrix.entry.hub || 'opensearchproject' }}, tests=${{ matrix.entry.tests || 'default' }})
     runs-on: ubuntu-latest
@@ -65,11 +70,6 @@ jobs:
         working-directory: tests/${{ matrix.entry.tests || 'default' }}
         run: docker compose up -d
 
-      - name: Get Container ID
-        id: container
-        run: |
-          echo "CONTAINER_ID=$(docker ps -aqf "ancestor=opensearchproject/opensearch:${{ matrix.entry.version }}")" >> $GITHUB_OUTPUT
-
       - name: Run Tests
         run: |
           npm run test:spec -- \
@@ -79,10 +79,12 @@ jobs:
             --tests=tests/${{ matrix.entry.tests || 'default' }}
 
       - name: Get Container Logs
-        if: failure() && steps.container.outputs.CONTAINER_ID
+        if: failure()
         run: |
-          echo Dumping logs from ${{ steps.container.outputs.CONTAINER_ID }} ...
-          docker logs ${{ steps.container.outputs.CONTAINER_ID }}
+          for container_id in $(docker ps -aqf "ancestor=opensearchproject/opensearch:${{ matrix.entry.version }}"); do \
+            echo Dumping logs from $container_id ... && \
+            docker logs $container_id \
+          ; done
 
       - name: Upload Test Coverage Results
         uses: actions/upload-artifact@v4
@@ -106,13 +108,14 @@ jobs:
         shell: bash -eo pipefail {0}
         run: |
           jq -sc '
-            map(to_entries) |
-            flatten |
-            group_by(.key) |
-            map({key: .[0].key, value: map(.value) | max}) |
-            from_entries |
-            .' $(find ./coverage -name "test-spec-coverage-*.json") > ./coverage/coverage.json
-
+            (map(.operations) | add | unique | length) as $total_operations_count |
+            (map(.evaluated_operations) | add | unique | length) as $evaluated_operations_count |
+            { 
+              total_operations_count: $total_operations_count,
+              evaluated_operations_count: $evaluated_operations_count,
+              evaluated_paths_pct: $evaluated_operations_count | (10000 * . / $total_operations_count | round / 100)
+            }
+          ' $(find ./coverage -name "test-spec-coverage-*.json") > ./coverage/coverage.json
           cat ./coverage/coverage.json
 
       - name: Construct Comment Data Payload

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -71,14 +71,32 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Added API spec for security plugin  ([#271](https://github.com/opensearch-project/opensearch-api-specification/pull/271))
 - Added `/_plugins/_security/api/certificates/` ([#439](https://github.com/opensearch-project/opensearch-api-specification/pull/439))
 - Added `/_plugins/_ml/models/{model_id}/_deploy`, `_undeploy` and `knn_vector` type in `passage_embedding` ([#504](https://github.com/opensearch-project/opensearch-api-specification/pull/504))
-- Added `PersonalizeSearchRanking`, `RetrievalAugmentedGeneration`, `Rerank`, `Collapse`, `TruncateHits` and `SplitResponseProcessor` ([#505](https://github.com/opensearch-project/opensearch-api-specification/pull/505))
+- Added `PersonalizeSearchRanking`, `RetrievalAugmentedGeneration`, `Rerank`, `Collapse`, `TruncateHits`, `SortResponseProcessor` and `SplitResponseProcessor` ([#505](https://github.com/opensearch-project/opensearch-api-specification/pull/505))
 - Added `/_plugins/_security/api/certificates/` to API spec  ([#439](https://github.com/opensearch-project/opensearch-api-specification/pull/439))
 - Added support for annotating and testing the API spec against multiple OpenSearch distributions ([#483](https://github.com/opensearch-project/opensearch-api-specification/pull/483))
 - Added `read_time`, `write_time`, `queue_size` and `io_time_in_millis` to `IoStatDevice` ([#483](https://github.com/opensearch-project/opensearch-api-specification/pull/483))
 - Added `total_rejections_breakup` to `ShardIndexingPressureStats` ([#483](https://github.com/opensearch-project/opensearch-api-specification/pull/483))
 - Added `cancelled_task_percentage` and `current_cancellation_eligible_tasks_count` to `ShardSearchBackpressureTaskCancellationStats` ([#483](https://github.com/opensearch-project/opensearch-api-specification/pull/483))
 - Added detailed test coverage report ([#513](https://github.com/opensearch-project/opensearch-api-specification/pull/513))
 - Added 404 responses to `/_alias/{name}` and `/{index}/_alias/{name}` ([#519](https://github.com/opensearch-project/opensearch-api-specification/pull/519))
+- Added `asynchronous_search` ([#525](https://github.com/opensearch-project/opensearch-api-specification/pull/525))
+- Added `DELETE /_plugins/_ml/tasks/{task_id}` ([#530](https://github.com/opensearch-project/opensearch-api-specification/pull/530))
+- Added `/_plugins/_flow_framework` ([#508](https://github.com/opensearch-project/opensearch-api-specification/issues/508))
+- Added `AwarenessAttributeStats` to `/_cluster/health` ([#534](https://github.com/opensearch-project/opensearch-api-specification/pull/534))
+- Added `cache_reserved_in_bytes` to `ClusterFileSystem` ([#534](https://github.com/opensearch-project/opensearch-api-specification/pull/534))
+- Added `cluster_manager` to `ClusterNodeCount` ([#534](https://github.com/opensearch-project/opensearch-api-specification/pull/534))
+- Added support for `query` with `terms` in `_search` ([#546](https://github.com/opensearch-project/opensearch-api-specification/pull/546))([#564](https://github.com/opensearch-project/opensearch-api-specification/pull/564))([#570](https://github.com/opensearch-project/opensearch-api-specification/pull/570))
+- Added `h`, `bytes`, `time` parameters to cat API tests ([#551](https://github.com/opensearch-project/opensearch-api-specification/pull/551))
+- Added `terminated_early` to `count@200` ([#547](https://github.com/opensearch-project/opensearch-api-specification/pull/547))
+- Added request and response schemas for `/_cluster/routing/awareness/{attribute}/weights` ([#524](https://github.com/opensearch-project/opensearch-api-specification/pull/524))
+- Added request and response schemas for `/_cluster/decommission/awareness` ([#524](https://github.com/opensearch-project/opensearch-api-specification/pull/524))
+- Added `Bytes` component of type number ([#552](https://github.com/opensearch-project/opensearch-api-specification/pull/552))
+- Added `xy_shape` query ([#531](https://github.com/opensearch-project/opensearch-api-specification/pull/531))
+- Added `/_plugins/_flow_framework/`, `_search`, `state/_search`, `_provision`, `_deprovision`, `_steps`, and `_status` ([#508](https://github.com/opensearch-project/opensearch-api-specification/issues/508)) ([#833](https://github.com/opensearch-project/flow-framework/issues/833))
+- Added `/_plugins/_ism/policies`, `add`, `remove`, `change_policy`, `explain`, and `retry` ([#568](https://github.com/opensearch-project/opensearch-api-specification/pull/568)) ([#578](https://github.com/opensearch-project/opensearch-api-specification/pull/578))
+- Added `/_plugins/refresh_search_analyzers` ([#578](https://github.com/opensearch-project/opensearch-api-specification/pull/578))
+- Added `/_plugins/_ml/agents/_register`, `/_plugins/_ml/connectors/_create`, `DELETE /_plugins/_ml/agents/{agent_id}`, `DELETE /_plugins/_ml/connectors/{connector_id}` ([#228](https://github.com/opensearch-project/opensearch-api-specification/issues/228))
+- Added the `context` query param to the `put_script` APIs ([#586](https://github.com/opensearch-project/opensearch-api-specification/pull/586))
 
 ### Changed
 
@@ -87,17 +105,29 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Renamed `main` release tag to `main-latest` ([#321](https://github.com/opensearch-project/opensearch-api-specification/pull/321))
 - Replaced usages of `Opensearch` with `OpenSearch` ([#335](https://github.com/opensearch-project/opensearch-api-specification/pull/335))
 - Prevented merger tool from printing warnings when used by tester tool ([#359](https://github.com/opensearch-project/opensearch-api-specification/pull/359))
-- Replaced the deprecated fs.rmdirSync with fs.rmSync ([#359](https://github.com/opensearch-project/opensearch-api-specification/pull/359))
-- Tester tool now provides better context for non-2XX responses when --verbose is used ([#359](https://github.com/opensearch-project/opensearch-api-specification/pull/359))
-- Lock testing for next release of OpenSearch to a specific SHA ([#431](https://github.com/opensearch-project/opensearch-api-specification/pull/431))
-- Replace nullable with null type ([#436](https://github.com/opensearch-project/opensearch-api-specification/pull/436))
+- Replaced the deprecated `fs.rmdirSync` with `fs.rmSync` ([#359](https://github.com/opensearch-project/opensearch-api-specification/pull/359))
+- Added better context for non-2XX responses when `--verbose` is used with the tester tool ([#359](https://github.com/opensearch-project/opensearch-api-specification/pull/359))
+- Locked testing for next release of OpenSearch to a specific SHA ([#431](https://github.com/opensearch-project/opensearch-api-specification/pull/431))
+- Replaced nullable with `null` type ([#436](https://github.com/opensearch-project/opensearch-api-specification/pull/436))
 - Split test suite ([#472])(https://github.com/opensearch-project/opensearch-api-specification/pull/472)
+- Changed `WriteResponseBase`'s `_primary_term`, `_seq_no` & `_version` to have `int64` format ([#530](https://github.com/opensearch-project/opensearch-api-specification/pull/530))
+- Adjusted indices, shards cat API to test against unassigned indices ([#551](https://github.com/opensearch-project/opensearch-api-specification/pull/551))
+- Corrected Sort schema for `/_search`'s request body ([#529](https://github.com/opensearch-project/opensearch-api-specification/issues/529))
+- Rename `Bytes` component to `StorageType` ([#552](https://github.com/opensearch-project/opensearch-api-specification/pull/552))
+- Rename `ByteSize` to `StorageSize` ([#552](https://github.com/opensearch-project/opensearch-api-specification/pull/552))
+- Split `Percentage` into `PercentageNumber`, `PercentageString` ([#573](https://github.com/opensearch-project/opensearch-api-specification/pull/573))
+- Unified all `routing` query params to take a string or array of strings ([#586](https://github.com/opensearch-project/opensearch-api-specification/pull/586))
+- Changed `rank_eval`'s `search_type` query param to use the `SearchType` enum rather than a plain string ([#586](https://github.com/opensearch-project/opensearch-api-specification/pull/586))
+- Changed `cluster.reroute`'s `metric` path param to use an enum of metrics ([#586](https://github.com/opensearch-project/opensearch-api-specification/pull/586))
+- Changed `indices.stats`'s `metric` path param to use an enum of metrics ([#586](https://github.com/opensearch-project/opensearch-api-specification/pull/586))
+- Changed `CleanupRepositoryResults`' properties to be `int64`s ([#587](https://github.com/opensearch-project/opensearch-api-specification/pull/587))
 
 ### Deprecated
 
 ### Removed
 
 - Removed the ability to skip an individual spec test ([#358](https://github.com/opensearch-project/opensearch-api-specification/pull/358))
+- Removed `shape` query ([#531](https://github.com/opensearch-project/opensearch-api-specification/pull/531))
 
 ### Fixed
 
@@ -125,6 +155,17 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Fixed security spec to add support for 400 and 403s ([#439](https://github.com/opensearch-project/opensearch-api-specification/pull/439))
 - Fixed required parameters in `NodeInfo` and `NodeOperatingSystemInfo` ([#483](https://github.com/opensearch-project/opensearch-api-specification/pull/483))
 - Fixed query DSL `neural` field `query_image` set `contentEncoding` and `model_id` as optional ([#512](https://github.com/opensearch-project/opensearch-api-specification/pull/512))
+- Fixed `knn` query specification ([#538](https://github.com/opensearch-project/opensearch-api-specification/pull/538))([#580](https://github.com/opensearch-project/opensearch-api-specification/pull/580))
+- Fixed content-type for `/hot_threads` ([#543](https://github.com/opensearch-project/opensearch-api-specification/pull/543))
+- Fixed `geo_distance` query spec ([#560](https://github.com/opensearch-project/opensearch-api-specification/pull/560))
+- Fixed `/_cluster/settings` returning flat results ([#545](https://github.com/opensearch-project/opensearch-api-specification/pull/545))
+- Fixed missing fields in `_cat` API ([#551](https://github.com/opensearch-project/opensearch-api-specification/pull/551))
+- Fixed `geo_distance` query spec ([#561](https://github.com/opensearch-project/opensearch-api-specification/pull/561))
+- Fixed `geo_bounding_box` and `geo_shape` queries ([#531](https://github.com/opensearch-project/opensearch-api-specification/pull/531))
+- Fixed tasks namespace schemas ([#520](https://github.com/opensearch-project/opensearch-api-specification/pull/520))
+- Fixed `/_plugins/_transform/_preview` ([#568](https://github.com/opensearch-project/opensearch-api-specification/pull/568))
+- Fixed create/delete/index operation in `_bulk` ([#582](https://github.com/opensearch-project/opensearch-api-specification/pull/582))
+- Add `mode` and `compression` to k-NN index creation and search, and add `rescore` and `oversample_factor` to k-NN search ([#588](https://github.com/opensearch-project/opensearch-api-specification/pull/588))
 
 ### Security
 

diff --git a/DEVELOPER_GUIDE.md b/DEVELOPER_GUIDE.md
@@ -17,15 +17,15 @@
     * [Spec Linter](#spec-linter)
       * [Arguments](#arguments-1)
       * [Example](#example-1)
+    * [Spec Tester](#spec-tester)
     * [Dump Cluster Spec](#dump-cluster-spec)
       * [Arguments](#arguments-2)
       * [Example](#example-2)
     * [Coverage](#coverage)
       * [Arguments](#arguments-3)
       * [Example](#example-3)
-    * [Testing](#testing)
-      * [Tests](#tests)
-      * [Lints](#lints)
+    * [Tools Testing](#tools-testing)
+    * [Tools Linting](#tools-linting)
   * [Workflows](#workflows)
     * [Analyze PR Changes](#analyze-pr-changes)
     * [Build](#build)
@@ -211,6 +211,14 @@ We can take advantage of the default values and simply lint the specification vi
 npm run lint:spec
 ```
 
+### Spec Tester
+
+```bash
+npm run test:spec -- --help
+```
+
+The spec test framework validates the OpenSearch spec against a running OpenSearch cluster. As you modify the spec, you should add or update the spec test stories in the [./tests](tests) directory. For information on this topic, see [TESTING_GUIDE.md](TESTING_GUIDE.md).
+
 ### [Dump Cluster Spec](tools/src/dump-cluster-spec)
 
 ```bash
@@ -306,9 +314,7 @@ The output file `build/coverage.json` will now contain data of like below:
 }
 ```
 
-### Testing
-
-#### Tests
+### Tools Testing
 
 All tools should have tests added in [tools/tests](tools/tests), tests are implemented using [Jest](https://jestjs.io/). They can be run via:
 ```bash
@@ -322,7 +328,7 @@ npm run jest -- tools/tests/linter/lint.test.ts
 
 The test suite contains unit tests and integration tests. Integration tests, such as [these](tools/tests/tester/integ/), require a local instance of OpenSearch and are placed into a folder named `integ`. Unit tests are run in parallel and integration tests are run sequentially using `--runInBand`. You can run unit tests with `npm run test:unit` separately from integration tests with `npm run test:integ`.
 
-#### Lints
+### Tools Linting
 
 All TypeScript code and YAML files are linted using [ESLint](https://eslint.org/), [typescript-eslint](https://typescript-eslint.io/), and [eslint-plugin-yml](https://ota-meshi.github.io/eslint-plugin-yml/). Linting can be run via:
 ```bash