Merge remote-tracking branch 'refs/remotes/origin/sql-namespace' into…

… sql-namespace

.cspell

@@ -47,12 +47,14 @@ aarch
 actiongroup
 actiongroups
 aggregatable
+argjson
 asciifolding
 authc
 authinfo
 authtoken
 autocut
 backendroles
+backpressure
 beider
 boxplot
 caverphone
@@ -124,6 +126,7 @@ nysiis
 opendistro
 opensearch
 opensearchproject
+opensearchstaging
 ords
 oversample
 performanceanalyzer
@@ -164,6 +167,7 @@ termvectors
 tfidf
 tokenfilters
 translog
+tubone
 unigrams
 unmatch
 untriaged

.github/opensearch-cluster/docker-compose.yml

@@ -2,11 +2,12 @@ version: '3'
 
 services:
   opensearch-cluster:
-    image: opensearchproject/opensearch:${OPENSEARCH_VERSION:-latest}
+    image: ${OPENSEARCH_DOCKER_HUB_PROJECT:-opensearchproject}/opensearch:${OPENSEARCH_VERSION:-latest}${OPENSEARCH_DOCKER_REF}
     ports:
-      - "9200:9200"
-      - "9600:9600"
+      - '9200:9200'
+      - '9600:9600'
     environment:
+      - 'OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!}'
       - discovery.type=single-node
-      - "OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!}"
-      - "path.repo=/tmp/opensearch/repo"
+      - path.repo=/tmp/opensearch/repo
+      - plugins.index_state_management.job_interval=1

.github/pr-comment-templates/pr-test-coverage-analysis.template.md

@@ -0,0 +1,8 @@
+## Spec Test Coverage Analysis
+{{with .test_coverage}}
+
+| Total             | Tested                                                   |
+|-------------------|----------------------------------------------------------|
+| {{.paths_count}}  | {{.evaluated_paths_count}} ({{.evaluated_paths_pct}} %)  |
+
+{{end}}

.github/workflows/build.yml

@@ -28,13 +28,42 @@ jobs:
         shell: bash
         run: echo "name=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT
 
+      - name: Upload Artifacts
+        id: upload
+        uses: actions/upload-artifact@v4
+        with:
+          name: build
+          path: |
+            build/*
+
       - name: Release Specification to GitHub
+        id: release
         uses: marvinpinto/[email protected]
         with:
-          repo_token: "${{ secrets.GITHUB_TOKEN }}"
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
           automatic_release_tag: ${{ steps.branch.outputs.name }}-latest
           prerelease: true
           title: OpenSearch OpenAPI Spec (${{ steps.branch.outputs.name }})
           files: |
             LICENSE.txt
-            build/*
+            build/*
+
+      - name: Extract Changelog
+        id: changelog
+        uses: sean0x42/markdown-extract@v2
+        with:
+          file: CHANGELOG.md
+          pattern: Unreleased
+
+      - name: Update Release Description
+        id: update
+        uses: tubone24/[email protected]
+        env:
+          TAG_NAME: ${{ steps.branch.outputs.name }}-latest
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          body: |
+            ${{ steps.changelog.outputs.markdown }}
+
+            ### Links
+            - [Build Artifact](${{ steps.upload.outputs.artifact-url }})

.github/workflows/changelog.yml

@@ -1,7 +1,7 @@
 name: Verify Changelog
 on:
   pull_request:
-    types: [opened, edited, review_requested, synchronize, reopened, ready_for_review, labeled, unlabeled]
+    types: [edited, labeled, opened, ready_for_review, reopened, review_requested, synchronize, unlabeled]
 
 jobs:
   verify-changelog:
@@ -14,4 +14,4 @@ jobs:
 
       - uses: dangoslen/changelog-enforcer@v3
         with:
-          skipLabels: "autocut, skip-changelog"
+          skipLabels: 'autocut, skip-changelog'

.github/workflows/check-links.yml

@@ -1,6 +1,6 @@
 name: Check Links
 
-on: [push, pull_request]
+on: [pull_request,push]
 
 jobs:
   check:

.github/workflows/lint.yml

@@ -0,0 +1,31 @@
+name: Lint
+
+on:
+  push:
+    branches: ['**']
+    paths:
+      - '**/*.ts'
+      - '**/*.yaml'
+  pull_request:
+    branches: ['**']
+    paths:
+      - '**/*.ts'
+      - '**/*.yaml'
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v3
+        with:
+          node-version: '20'
+
+      - name: Install Dependencies
+        run: npm install
+
+      - name: Lint
+        run: npm run lint

.github/workflows/pr-comment.yml

@@ -4,6 +4,7 @@ on:
   workflow_run:
     workflows:
       - Analyze PR Changes
+      - Test Spec
     types:
       - completed
 
@@ -71,4 +72,4 @@ jobs:
           comment-id: ${{ steps.fc.outputs.comment-id }}
           issue-number: ${{ env.PR_NUMBER }}
           body: ${{ steps.render.outputs.result }}
-          edit-mode: replace
+          edit-mode: replace

.github/workflows/test-spec.yml

@@ -2,26 +2,41 @@ name: Test Spec
 
 on:
   push:
-    branches: ['**']
     paths:
-      - 'package*.json'
+      - .github/workflows/test-spec.yml
+      - package*.json
+      - spec/**
+      - tests/**
+      - tools/src/tester/**
       - tsconfig.json
-      - 'tools/src/tester/**'
-      - 'spec/**'
   pull_request:
-    branches: ['**']
     paths:
-      - 'package*.json'
+      - .github/workflows/test-spec.yml
+      - package*.json
+      - spec/**
+      - tests/**
+      - tools/src/tester/**
       - tsconfig.json
-      - 'tools/src/tester/**'
-      - 'spec/**'
 
 jobs:
   test-opensearch-spec:
+    strategy:
+      matrix:
+        entry:
+          - {version: 1.3.17, admin_password: admin}
+          - {version: 2.0.0, admin_password: admin}
+          - {version: 2.15.0}
+          - {version: 2.16.0, hub: opensearchstaging, ref: '@sha256:50fbfe3b95c41e92a113ada3e80513ba4524dfc8a25dc6aaeff2bbe1e1145d5f'}
+
+    name: test-opensearch-spec (version=${{ matrix.entry.version }}, hub=${{ matrix.entry.hub || 'opensearchproject' }})
     runs-on: ubuntu-latest
+
     env:
-      OPENSEARCH_VERSION: 2.12.0
-      OPENSEARCH_PASSWORD: myStrongPassword123!
+      OPENSEARCH_DOCKER_HUB_PROJECT: ${{ matrix.entry.hub || 'opensearchproject' }}
+      OPENSEARCH_DOCKER_REF: ${{ matrix.entry.ref }}
+      OPENSEARCH_VERSION: ${{ matrix.entry.version }}
+      OPENSEARCH_PASSWORD: ${{ matrix.entry.admin_password || 'myStrongPassword123!' }}
+
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v4
@@ -36,7 +51,64 @@ jobs:
 
       - name: Run OpenSearch Cluster
         working-directory: .github/opensearch-cluster
-        run: docker-compose up -d && sleep 60
+        run: docker-compose up -d
 
       - name: Run Tests
-        run: npm run test:spec -- --opensearch-insecure
+        run: |
+          npm run test:spec -- --opensearch-insecure --coverage coverage/test-spec-coverage-${{ matrix.entry.version }}.json
+
+      - name: Upload Test Coverage Results
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-${{ matrix.entry.version }}
+          path: coverage/test-spec-coverage-${{ matrix.entry.version }}.json
+
+  merge-coverage:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'pull_request'
+    needs: test-opensearch-spec
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Download Spec Coverage Data
+        uses: actions/download-artifact@v4
+        with:
+          path: coverage
+
+      - name: Combine Test Coverage Data
+        shell: bash -eo pipefail {0}
+        run: |
+          jq -sc '
+            map(to_entries) |
+            flatten |
+            group_by(.key) |
+            map({key: .[0].key, value: map(.value) | max}) |
+            from_entries |
+            .' $(find ./coverage -name "test-spec-coverage-*.json") > ./coverage/coverage.json
+
+          cat ./coverage/coverage.json
+
+      - name: Construct Comment Data Payload
+        shell: bash -eo pipefail {0}
+        run: |
+          jq \
+            --arg pr_number ${PR_NUMBER} \
+            --slurpfile test_coverage ./coverage/coverage.json \
+            --null-input '
+            {
+              "pr_number": ($pr_number),
+              "comment_identifier": "## Spec Test Coverage Analysis",
+              "template_name": "pr-test-coverage-analysis",
+              "template_data": {
+                "test_coverage": ($test_coverage[0])
+              }
+            }
+            ' | tee pr-comment.json
+        env:
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+
+      - name: Upload PR Comment Payload
+        uses: actions/upload-artifact@v4
+        with:
+          name: pr-comment
+          path: pr-comment.json

.github/workflows/test-tools-integ.yml

@@ -4,27 +4,27 @@ on:
   push:
     branches: ['**']
     paths:
-      - 'package*.json'
+      - '.github/workflows/test-tools*.yml'
       - eslint.config.mjs
       - jest.config.js
-      - tsconfig.json
+      - 'package*.json'
       - 'tools/**'
-      - '.github/workflows/test-tools*.yml'
+      - tsconfig.json
   pull_request:
     branches: ['**']
     paths:
-      - 'package*.json'
+      - '.github/workflows/test-tools*.yml'
       - eslint.config.mjs
       - jest.config.js
-      - tsconfig.json
+      - 'package*.json'
       - 'tools/**'
-      - '.github/workflows/test-tools*.yml'
+      - tsconfig.json
 
 jobs:
   test:
     runs-on: ubuntu-latest
     env:
-      OPENSEARCH_VERSION: 2.12.0
+      OPENSEARCH_VERSION: 2.15.0
       OPENSEARCH_PASSWORD: myStrongPassword123!
       OPENSEARCH_URL: https://localhost:9200
     steps:
@@ -35,7 +35,7 @@ jobs:
         working-directory: .github/opensearch-cluster
         run: |
           docker-compose up -d
-          sleep 30
+          sleep 15
 
       - name: Setup Node.js
         uses: actions/setup-node@v3

.github/workflows/test-tools-unit.yml

@@ -4,21 +4,21 @@ on:
   push:
     branches: ['**']
     paths:
+      - '.github/workflows/test-tools*.yml'
+      - eslint.config.mjs
+      - jest.config.js
       - 'package*.json'
-      - 'eslint.config.mjs'
-      - 'jest.config.js'
-      - 'tsconfig.json'
       - 'tools/**'
-      - '.github/workflows/test-tools*.yml'
+      - tsconfig.json
   pull_request:
     branches: ['**']
     paths:
+      - '.github/workflows/test-tools*.yml'
+      - eslint.config.mjs
+      - jest.config.js
       - 'package*.json'
-      - 'eslint.config.mjs'
-      - 'jest.config.js'
-      - 'tsconfig.json'
       - 'tools/**'
-      - '.github/workflows/test-tools*.yml'
+      - tsconfig.json
 
 jobs:
   test:
@@ -35,9 +35,6 @@ jobs:
       - name: Install Dependencies
         run: npm install
 
-      - name: Lint
-        run: npm run lint
-
       - name: Tests
         run: |
           npm run test:unit -- --coverage

.gitignore

@@ -17,5 +17,5 @@ _site/
 build/
 
 # coverage output
-coverage/
+/coverage/