Merge branch 'opensearch-project:main' into observation_api

.github/workflows/changelog.yml

@@ -7,7 +7,7 @@ jobs:
   verify-changelog:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           ref: ${{ github.event.pull_request.head.sha }}

.github/workflows/test-spec.yml

@@ -27,17 +27,17 @@ jobs:
             admin_password: admin
           - version: 2.0.0
             admin_password: admin
-          - version: 2.15.0
-          - version: 2.15.0
+          - version: 2.16.0
+          - version: 2.16.0
             tests: plugins/index_state_management
-          - version: 2.15.0
-            tests: snapshot
           - version: 2.16.0
+            tests: snapshot
+          - version: 2.17.0
             hub: opensearchstaging
-            ref: '@sha256:bcd7f5d5d30231f24f266064248cc8d3306574948190f7bf93016dff29acf17e'
+            ref: '@sha256:6398c27d7560626ed6b0ba28b3d6b20b7f00c6d94abf45ad3a820f8eeb3d61a3'
           - version: 3.0.0
             hub: opensearchstaging
-            ref: '@sha256:db1918b2b8f7ef6c22dd6ff54a0640877c3d395a392a53864745024933981e3b'
+            ref: '@sha256:101681eea630393f8caf5987dd023a975a9656b63090a07bfdfe6ad2f73f0640'
 
     name: test-opensearch-spec (version=${{ matrix.entry.version }}, hub=${{ matrix.entry.hub || 'opensearchproject' }}, tests=${{ matrix.entry.tests || 'default' }})
     runs-on: ubuntu-latest
@@ -47,7 +47,7 @@ jobs:
       OPENSEARCH_DOCKER_REF: ${{ matrix.entry.ref }}
       OPENSEARCH_VERSION: ${{ matrix.entry.version }}
       OPENSEARCH_PASSWORD: ${{ matrix.entry.admin_password || 'myStrongPassword123!' }}
-      OPENSEARCH_JAVA_OPTS: ${{ matrix.entry.opts }} -Xms2g -Xmx2g
+      OPENSEARCH_JAVA_OPTS: ${{ matrix.entry.opts }} -Xms8g -Xmx8g
 
     steps:
       - name: Checkout Repo
@@ -95,7 +95,7 @@ jobs:
     if: github.event_name == 'pull_request'
     needs: test-opensearch-spec
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Download Spec Coverage Data
         uses: actions/download-artifact@v4

.github/workflows/test-tools-integ.yml

@@ -24,7 +24,7 @@ jobs:
   test:
     runs-on: ubuntu-latest
     env:
-      OPENSEARCH_VERSION: 2.15.0
+      OPENSEARCH_VERSION: 2.16.0
       OPENSEARCH_PASSWORD: myStrongPassword123!
       OPENSEARCH_URL: https://localhost:9200
     steps:

CHANGELOG.md

@@ -108,7 +108,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Fixed `/_mapping` with `index` in query ([#385](https://github.com/opensearch-project/opensearch-api-specification/pull/385))
 - Fixed duplicate `/_nodes/{node_id}` path ([#416](https://github.com/opensearch-project/opensearch-api-specification/pull/416))
 - Fixed `_source` accepting an array of fields in `/_search` ([#430](https://github.com/opensearch-project/opensearch-api-specification/pull/430))
-- Fixed `_update_by_query` with a simple term ([451](https://github.com/opensearch-project/opensearch-api-specification/pull/451))
+- Fixed `_update_by_query` with a simple term ([#451](https://github.com/opensearch-project/opensearch-api-specification/pull/451))
+- Fixed `Duration` to allow for non-integers ([#479](https://github.com/opensearch-project/opensearch-api-specification/pull/479))
 
 ### Security
 

TESTING_GUIDE.md

@@ -6,6 +6,7 @@
     - [Common Errors](#common-errors)
       - [401 Unauthorized](#401-unauthorized)
       - [FORBIDDEN/10/cluster create-index blocked (api)](#forbidden10cluster-create-index-blocked-api)
+      - [FAILED  Cat with a json response (from security-analytics).](#failed--cat-with-a-json-response-from-security-analytics)
   - [Writing Spec Tests](#writing-spec-tests)
     - [Simple Test Story](#simple-test-story)
     - [Using Output from Previous Chapters](#using-output-from-previous-chapters)
@@ -85,6 +86,9 @@ curl -k -X PUT --user "admin:${OPENSEARCH_PASSWORD}" https://localhost:9200/_clu
 '
 ```
 
+#### FAILED  Cat with a json response (from security-analytics).
+The cluster is not loading plugins correctly, maybe it was stopped using `docker kill` instead of `docker stop`. Recreating the cluster should fix the issue: `docker compose up --force-recreate -d`.
+
 ## Writing Spec Tests
 
 The spec tests reside in the [tests/](tests) directory. Tests are organized in suites ([default](tests/default/), etc.), and subsequently in folders that match [namespaces](spec/namespaces). For example, tests for APIs defined in [spec/namespaces/indices.yaml](spec/namespaces/indices.yaml) can be found in [tests/default/indices/index.yaml](tests/default/indices/index.yaml) (for `/{index}`), and [tests/default/indices/doc.yaml](tests/default/indices/doc.yaml) (for `/{index}/_doc`). 

package.json

@@ -34,6 +34,7 @@
     "@types/titlecase": "^1.1.2",
     "@types/tmp": "^0.2.6",
     "@typescript-eslint/eslint-plugin": "^6.21.0",
+    "axios-mock-adapter": "^2.0.0",
     "ajv": "^8.13.0",
     "ajv-errors": "^3.0.0",
     "ajv-formats": "^3.0.1",

spec/_info.yaml

@@ -2,4 +2,4 @@ $schema: ./json_schemas/_info.schema.yaml
 
 title: OpenSearch API Specification
 version: 1.0.0
-x-api-version: 2.15.0
+x-api-version: 2.16.0

spec/schemas/_common.yaml

@@ -38,7 +38,7 @@ components:
       description: |-
         A duration. Units can be `nanos`, `micros`, `ms` (milliseconds), `s` (seconds), `m` (minutes), `h` (hours) and
         `d` (days). Also accepts "0" without a unit and "-1" to indicate an unspecified value.
-      pattern: ^([0-9]+)(?:d|h|m|s|ms|micros|nanos)$
+      pattern: ^([0-9\.]+)(?:d|h|m|s|ms|micros|nanos)$
       type: string
     Metadata:
       type: object

tests/default/_core/reindex.yaml

@@ -39,6 +39,8 @@ chapters:
   - synopsis: Reindex a subset of documents (match).
     path: /_reindex
     method: POST
+    parameters:
+      refresh: true
     request:
       payload:
         source:
@@ -82,7 +84,7 @@ chapters:
     response:
       status: 200
       payload:
-        total: 1
+        total: 2
   - synopsis: Reindex only unique documents.
     path: /_reindex
     method: POST

tools/src/OpenSearchHttpClient.ts

@@ -133,18 +133,23 @@ export class OpenSearchHttpClient {
     this._opts = opts
     this._logger = opts?.logger ?? new Logger()
 
-    let auth = undefined
-    let sigv4_interceptor = undefined
+    let auth_middleware = undefined
 
     if (opts?.basic_auth !== undefined) {
       this._logger.info(`Authenticating with ${opts.basic_auth.username} ...`)
-      auth = opts.basic_auth
+      auth_middleware = ((request: any): any => {
+        if (request.headers.Authorization === undefined) {
+          const base64 = Buffer.from(`${opts.basic_auth?.username}:${opts.basic_auth?.password}`, 'utf8').toString('base64');
+          request.headers.Authorization = `Basic ${base64}`
+        }
+        return request
+      })
     } else if (opts?.aws_auth !== undefined) {
       this._logger.info(`Authenticating using SigV4 with ${opts.aws_auth.aws_access_key_id} (${opts.aws_auth.aws_region}) ...`)
-      sigv4_interceptor = aws4Interceptor({
+      auth_middleware = aws4Interceptor({
         options: {
           region: opts.aws_auth.aws_region,
-          service: 'es'
+          service: opts.aws_auth.aws_service
         },
         credentials: {
           accessKeyId: opts.aws_auth.aws_access_key_id,
@@ -158,13 +163,12 @@ export class OpenSearchHttpClient {
 
     this._axios = axios.create({
       baseURL: opts?.url ?? DEFAULT_URL,
-      auth,
       httpsAgent: new https.Agent({ rejectUnauthorized: !(opts?.insecure ?? DEFAULT_INSECURE) }),
       responseType: opts?.responseType,
     })
 
-    if (sigv4_interceptor !== undefined) {
-      this._axios.interceptors.request.use(sigv4_interceptor)
+    if (auth_middleware !== undefined) {
+      this._axios.interceptors.request.use(auth_middleware)
     }
   }
 

tools/src/tester/TestRunner.ts

@@ -20,10 +20,6 @@ import { OpenSearchHttpClient } from 'OpenSearchHttpClient'
 import * as ansi from './Ansi'
 import _ from 'lodash'
 
-const EXCLUDED_FILES = [
-  'docker-compose.yml'
-]
-
 export default class TestRunner {
   private readonly _http_client: OpenSearchHttpClient
   private readonly _story_validator: StoryValidator
@@ -69,6 +65,9 @@ export default class TestRunner {
     const path = file === '' ? folder : `${folder}/${file}`
     const next_prefix = prefix === '' ? file : `${prefix}/${file}`
     if (fs.statSync(path).isFile()) {
+      if (!path.endsWith('.yaml')) {
+        return []
+      }
       const story: Story = read_yaml(path)
       return [{
         display_path: next_prefix === '' ? basename(path) : next_prefix,
@@ -77,9 +76,7 @@ export default class TestRunner {
       }]
     } else {
       return _.compact(fs.readdirSync(path).flatMap(next_file => {
-        if (!EXCLUDED_FILES.includes(next_file)) {
-          return this.#collect_story_files(path, next_file, next_prefix)
-        }
+        return this.#collect_story_files(path, next_file, next_prefix)
       }))
     }
   }

tools/tests/tester/OpenSearchHttpClient.test.ts

@@ -9,58 +9,68 @@
 
 import axios from "axios";
 import { OpenSearchHttpClient } from "OpenSearchHttpClient"
-
-jest.mock('axios')
+import AxiosMockAdapter from "axios-mock-adapter";
 
 describe('OpenSearchHttpClient', () => {
-  let mocked_axios: jest.Mocked<typeof axios> = axios as jest.Mocked<typeof axios>
+  var mock = new AxiosMockAdapter(axios)
 
-  beforeEach(() => {
-    mocked_axios.create.mockReturnThis()
-    mocked_axios.interceptors.request.use = jest.fn().mockReturnValue({ headers: {} })
+  afterEach(() => {
+    mock.reset()
   })
 
-  afterEach(() => {
-    jest.clearAllMocks()
+  it('adds a Basic auth header', async () => {
+    let client = new OpenSearchHttpClient({
+      url: 'https://localhost:9200',
+      basic_auth: {
+        username: 'u',
+        password: 'p'
+      }
+    })
+
+    mock.onAny().reply((config) => {
+      expect(config.headers?.Authorization).toMatch(/^Basic /)
+      return [200, { called: true }]
+    })
+
+    expect((await client.get('/')).data).toEqual({ called: true })
   })
 
-  it('uses password authentication', () => {
-    new OpenSearchHttpClient({
+  it('allows to overwrite Authorization', async () => {
+    let client = new OpenSearchHttpClient({
       url: 'https://localhost:9200',
       basic_auth: {
-        username: 'admin',
-        password: 'password'
+        username: 'u',
+        password: 'p'
       }
     })
 
-    expect(mocked_axios.create.mock.calls[0][0]).toMatchObject({
-      auth: {
-        username: 'admin',
-        password: 'password'
-      },
-      baseURL: 'https://localhost:9200'
+    mock.onAny().reply((config) => {
+      expect(config.headers?.Authorization).toEqual('custom')
+      return [200, { called: true }]
     })
 
-    expect(mocked_axios.interceptors.request.use).not.toHaveBeenCalled()
+    expect((await client.get('/', { headers: { Authorization: 'custom' } })).data).toEqual({ called: true })
   })
 
-  it('assigns a request interceptor with SigV4 authentication', () => {
-    new OpenSearchHttpClient({
+  it('adds a Sigv4 header', async () => {
+    let client = new OpenSearchHttpClient({
       url: 'https://localhost:9200',
       aws_auth: {
         aws_access_key_id: 'key id',
         aws_access_secret_key: 'secret key',
         aws_access_session_token: 'session token',
-        aws_region: 'us-west-2',
+        aws_region: 'us-west-42',
         aws_service: 'aoss'
       }
     })
 
-    expect(mocked_axios.create.mock.calls[0][0]).toMatchObject({
-      auth: undefined,
-      baseURL: 'https://localhost:9200'
+    mock.onAny().reply((config) => {
+      expect(config.headers?.Authorization).toMatch(
+        /^AWS4-HMAC-SHA256 Credential=key id\/\d*\/us-west-42\/aoss\/aws4_request/
+      )
+      return [200, { called: true }]
     })
 
-    expect(mocked_axios.interceptors.request.use).toHaveBeenCalled()
+    expect((await client.get('/')).data).toEqual({ called: true })
   })
 })

tools/tests/tester/fixtures/evals/passed.yaml

@@ -175,7 +175,7 @@ chapters:
   - title: This GET /_cat/health should be skipped (> 2.999.0).
     overall:
       result: SKIPPED
-      message: Skipped because version 2.15.0 does not satisfy >= 2.999.0.
+      message: Skipped because version 2.16.0 does not satisfy >= 2.999.0.
 epilogues:
   - title: DELETE /books
     overall: