Merge remote-tracking branch 'upstream/main' into security-tests

.cspell

@@ -1,52 +1,8 @@
-APIV
-Boxplot
-Bobg
-Boundsdouble
-CBOR
-DNFOF
-Decompounder
-Downsample
-Downsampling
-Ewma
-Fielddata
-Formattable
-Fragmenter
-Geohex
-Gsub
-Intertransport
-Jelinek
-Kuromoji
-Lovins
-Lucene
-Millis
-Moneyball
-Multisearch
-Moneyball
-Nanos
-Nori
-ONNX
-OPENSEARCH
-OSCPU
-Oversample
-Rebalance
-Reindex
-Rethrottle
-Rolespan
-Rollup
-Rollups
-Slowlog
-Sorani
-Stringifiedboolean
-Stringifiedinteger
-Tfidf
-Tika
-Translog
-Unmanaged
-Yrtsd
 aarch
 actiongroup
 actiongroups
 aggregatable
+APIV
 argjson
 asciifolding
 authc
@@ -56,63 +12,92 @@ autocut
 backendroles
 backpressure
 beider
+Bobg
+Boundsdouble
 boxplot
+Boxplot
 caverphone
+CBOR
 charfilters
 chisquared
 chuhlomin
 cpuacct
+cuserr
 daitch
 dangoslen
 dashboardsinfo
+datarows
 decompounder
+Decompounder
 dedup
 determinized
+distilbert
+DNFOF
 docvalue
+Downsample
 downsampling
+Downsampling
 evals
 ewma
+Ewma
 faiss
 fielddata
+Fielddata
 forcemerge
+Formattable
 fragmenter
+Fragmenter
 freqs
 generateonbehalfoftoken
 geohash
 geohex
+Geohex
 geoip
 geotile
 gsub
+Gsub
 haasephonetik
 heteroscedastic
 homoscedastic
 hotthreads
+huggingface
 hybridfs
 integ
 internalusers
+Intertransport
+Jelinek
 kibanainfo
 kibanaserver
 koelnerphonetik
 kstem
 kuromoji
+Kuromoji
 languageset
+Lovins
 lucene
+Lucene
 lycheeverse
 marvinpinto
 metaphone
 mget
 millis
+Millis
 mlockall
 mmap
 mmapfs
 mmdb
 mokotoff
+Moneyball
+Moneyball
 msearch
+msmarco
 mtermvectors
 mult
+Multisearch
 multitenancy
 mxyz
 nanos
+Nanos
 nbest
 nfkc
 ngram
@@ -122,13 +107,18 @@ nodeattrs
 nodesdn
 noops
 nori
+Nori
 nysiis
+ONNX
 opendistro
 opensearch
+OPENSEARCH
 opensearchproject
 opensearchstaging
 ords
+OSCPU
 oversample
+Oversample
 performanceanalyzer
 permissionsinfo
 pipefail
@@ -137,38 +127,54 @@ prirep
 rawfile
 readingform
 rebalance
+Rebalance
 recoverysource
 reindex
+Reindex
 relo
 reloadcerts
 remotestore
 rethrottle
+Rethrottle
 rolesmapping
+Rolespan
 rollup
+Rollup
 rollups
+Rollups
 romaji
 roundtrips
+Rudnick
 ruleset
 scriptless
 securityconfig
 slowlog
+Slowlog
 slurpfile
 snapshotted
 softmax
+Sorani
 sslinfo
 stoptags
 stopwords
+Stringifiedboolean
+Stringifiedinteger
 subqueries
 subschemas
 subword
+syserr
 tdigest
 tenantinfo
 termvectors
 tfidf
+Tfidf
+Tika
 tokenfilters
 translog
+Translog
 tubone
 unigrams
+Unmanaged
 unmatch
 untriaged
 updateable
@@ -177,3 +183,4 @@ urldecode
 vectory
 whoamiprotected
 wordnet
+Yrtsd

.github/workflows/test-spec.yml

@@ -26,7 +26,7 @@ jobs:
           - {version: 1.3.17, admin_password: admin}
           - {version: 2.0.0, admin_password: admin}
           - {version: 2.15.0}
-          - {version: 2.16.0, hub: opensearchstaging, ref: '@sha256:aa99ce3e2c9c94cd934cf98bab753a9920bfdbbb74bdc932d1b443d31c18d6b2'}
+          - {version: 2.16.0, hub: opensearchstaging, ref: '@sha256:50fbfe3b95c41e92a113ada3e80513ba4524dfc8a25dc6aaeff2bbe1e1145d5f'}
 
     name: test-opensearch-spec (version=${{ matrix.entry.version }}, hub=${{ matrix.entry.hub || 'opensearchproject' }})
     runs-on: ubuntu-latest
@@ -51,11 +51,14 @@ jobs:
 
       - name: Run OpenSearch Cluster
         working-directory: .github/opensearch-cluster
-        run: docker-compose up -d
+        run: docker compose up -d
 
       - name: Run Tests
         run: |
-          npm run test:spec -- --opensearch-insecure --coverage coverage/test-spec-coverage-${{ matrix.entry.version }}.json
+          npm run test:spec -- \
+            --opensearch-insecure \
+            --opensearch-version=${{ matrix.entry.version }} \
+            --coverage coverage/test-spec-coverage-${{ matrix.entry.version }}.json
 
       - name: Upload Test Coverage Results
         uses: actions/upload-artifact@v4
@@ -97,7 +100,7 @@ jobs:
             --null-input '
             {
               "pr_number": ($pr_number),
-              "comment_identifier": "# Test Coverage Analysis",
+              "comment_identifier": "## Spec Test Coverage Analysis",
               "template_name": "pr-test-coverage-analysis",
               "template_data": {
                 "test_coverage": ($test_coverage[0])
@@ -111,4 +114,4 @@ jobs:
         uses: actions/upload-artifact@v4
         with:
           name: pr-comment
-          path: pr-comment.json
+          path: pr-comment.json

.github/workflows/test-tools-integ.yml

@@ -34,7 +34,7 @@ jobs:
       - name: Run OpenSearch Cluster
         working-directory: .github/opensearch-cluster
         run: |
-          docker-compose up -d
+          docker compose up -d
           sleep 15
 
       - name: Setup Node.js

CHANGELOG.md

@@ -35,11 +35,14 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Added AjvErrorsParser to print more informative error messages ([#364](https://github.com/opensearch-project/opensearch-api-specification/issues/364))
 - Added JsonSchemaValidator, a wrapper for AJV ([#364](https://github.com/opensearch-project/opensearch-api-specification/issues/364))
 - Added support for `application/cbor` responses ([#371](https://github.com/opensearch-project/opensearch-api-specification/pull/371))
+- Added `/_plugins/_sql`, `close`, `explain` and `stats` ([#379](https://github.com/opensearch-project/opensearch-api-specification/pull/379))
+- Added tests for SQL namespace ([#379](https://github.com/opensearch-project/opensearch-api-specification/pull/379))
 - Added support for `application/smile` responses ([#386](https://github.com/opensearch-project/opensearch-api-specification/pull/386))
 - Added `doc_status`, `remote_store`, `segment_replication` and `unreferenced_file_cleanups_performed` to `SegmentStats` ([#395](https://github.com/opensearch-project/opensearch-api-specification/pull/395))
 - Added `concurrent_query_*` and `search_idle_reactivate_count_total` fields to `SearchStats` ([#395](https://github.com/opensearch-project/opensearch-api-specification/pull/395))
 - Added `remote_store` to `TranslogStats` ([#395](https://github.com/opensearch-project/opensearch-api-specification/pull/395))
 - Added `file` to `/_cache/clear` and `/{index}/_cache/clear` ([#396](https://github.com/opensearch-project/opensearch-api-specification/pull/396))
+- Add `strict_allow_templates` option for the dynamic mapping parameter ([#408](https://github.com/opensearch-project/opensearch-api-specification/pull/408))
 - Added a workflow to run tests against the next version of OpenSearch ([#409](https://github.com/opensearch-project/opensearch-api-specification/pull/409))
 - Added support for skipping tests using semver range ([#410](https://github.com/opensearch-project/opensearch-api-specification/pull/410))
 - Added `cluster_manager_timeout` to `HEAD /{index}` ([#421](https://github.com/opensearch-project/opensearch-api-specification/pull/421))
@@ -51,6 +54,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Added missing variants of `indices.put_alias` ([#434](https://github.com/opensearch-project/opensearch-api-specification/pull/434)) 
 - Added `plugins` to NodeInfoSettings ([#442](https://github.com/opensearch-project/opensearch-api-specification/pull/442)) 
 - Added test coverage ([#443](https://github.com/opensearch-project/opensearch-api-specification/pull/443))
+- Added `--opensearch-version` to `merger` that excludes schema elements per semver ([#428](https://github.com/opensearch-project/opensearch-api-specification/pull/428))
+- Added `retry` to `tester` to support asynchronous tasks ([453](https://github.com/opensearch-project/opensearch-api-specification/pull/453))
 - Added tests for security APIs ([#439](https://github.com/opensearch-project/opensearch-api-specification/pull/439))
 
 ### Changed
@@ -91,6 +96,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Fixed `/_mapping` with `index` in query ([#385](https://github.com/opensearch-project/opensearch-api-specification/pull/385))
 - Fixed duplicate `/_nodes/{node_id}` path ([#416](https://github.com/opensearch-project/opensearch-api-specification/pull/416))
 - Fixed `_source` accepting an array of fields in `/_search` ([#430](https://github.com/opensearch-project/opensearch-api-specification/pull/430))
+- Fixed `_update_by_query` with a simple term ([451](https://github.com/opensearch-project/opensearch-api-specification/pull/451))
 
 ### Security
 

DEVELOPER_GUIDE.md

@@ -173,6 +173,7 @@ The merger tool merges the multi-file OpenSearch spec into a single file for pro
 
 - `--source <path>`: The path to the root folder of the multi-file spec, defaults to `<repository-root>/spec`.
 - `--output <path>`: The path to write the final merged spec to, defaults to `<repository-root>/build/opensearch-openapi.yaml`.
+- `--opensearch-version`: An optional target version of OpenSearch, checking values of `x-version-added` and `x-version-removed`.
 
 #### Example
 
@@ -181,6 +182,12 @@ We can take advantage of the default values and simply merge the specification v
 npm run merge
 ```
 
+To generate a spec that does not contain any APIs or fields removed in version 2.0 (e.g. document `_type` fields).
+
+```bash
+npm run merge -- --opensearch-version=2.0
+```
+
 ### [Spec Linter](tools/src/linter)
 
 ```bash

TESTING_GUIDE.md

@@ -5,6 +5,7 @@
     - [Simple Test Story](#simple-test-story)
     - [Using Output from Previous Chapters](#using-output-from-previous-chapters)
     - [Managing Versions](#managing-versions)
+    - [Waiting for Tasks](#waiting-for-tasks)
 <!-- TOC -->
 
 # Spec Testing Guide
@@ -19,7 +20,7 @@ Set up an OpenSearch cluster with Docker:
 ```bash
 export OPENSEARCH_PASSWORD=<<your_password>>
 cd .github/opensearch-cluster
-docker-compose up -d
+docker compose up -d
 ```
 
 Run the tests (use `--opensearch-insecure` for a local cluster running in Docker that does not have a valid SSL certificate):
@@ -153,3 +154,24 @@ It's common to add a feature to the next version of OpenSearch. When adding a ne
 ```
 
 The [integration test workflow](.github/workflows/test-spec.yml) runs a matrix of OpenSearch versions, including the next version. Please check whether the workflow needs an update when adding version-specific tests.
+
+### Waiting for Tasks
+
+Some APIs behave asynchronously and may require a test to wait for a task to complete. This can be achived with a combination of `payload` and `retry`. 
+
+For example, an ML task returns `CREATED` when created, and `COMPLETED` when it's done. The example below will retry 3 times with an interval of 30 seconds until the task is complete. The default wait time is 1s.
+
+```yaml
+  - synopsis: Wait for task.
+    path: /_plugins/_ml/tasks/{task_id}
+    method: GET
+    parameters:
+      task_id: ${create_model.task_id}
+    response:
+      status: 200
+      payload:
+        state: COMPLETED
+    retry:
+      count: 3
+      wait: 30000
+```

eslint.config.mjs

@@ -56,7 +56,7 @@ export default [
         { selector: 'typeProperty', format: null }
       ],
       '@typescript-eslint/no-confusing-void-expression': 'error',
-      '@typescript-eslint/no-dynamic-delete': 'error',
+      '@typescript-eslint/no-dynamic-delete': 'off',
       '@typescript-eslint/no-invalid-void-type': 'error',
       '@typescript-eslint/no-non-null-assertion': 'error',
       '@typescript-eslint/no-unnecessary-type-assertion': 'error',