Added support for using a certificate and key. (#731)

Signed-off-by: dblock <[email protected]>

.github/workflows/test-spec.yml

@@ -53,6 +53,10 @@ jobs:
             tests: plugins/workload-management
           - version: 2.18.0
             tests: plugins/analysis
+          - version: 2.18.0
+            tests: plugins/security
+            cert: tests/plugins/security/.kirk.pem
+            key: tests/plugins/security/.kirk-key.pem
           - version: 2.19.0
             hub: opensearchstaging
             ref: '@sha256:4da23e0137b2b67206d23b36fcf0914cc39b3bf19310c782f536e4934b86f6cc'
@@ -97,6 +101,8 @@ jobs:
             --opensearch-version=${{ matrix.entry.version }} \
             --coverage coverage/test-spec-coverage-${{ steps.tests.outputs.hash }}.json \
             --opensearch-url=${{ matrix.entry.url || 'https://localhost:9200'}} \
+            --opensearch-cert=${{ matrix.entry.cert }} \
+            --opensearch-key=${{ matrix.entry.key }} \
             --tests=tests/${{ matrix.entry.tests || 'default' }}
 
       - name: Get Container Logs

CHANGELOG.md

@@ -33,6 +33,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Added response schema for `PUT` and `DELETE /_plugins/_transform/{id}` ([#722](https://github.com/opensearch-project/opensearch-api-specification/pull/716))
 - Added response schema for `GET /_plugins/_knn/warmup/{index}` ([#717](https://github.com/opensearch-project/opensearch-api-specification/pull/717))
 - Added support for multiple test verbs ([#724](https://github.com/opensearch-project/opensearch-api-specification/pull/724))
+- Added support for using a certificate and key in tests ([#731](https://github.com/opensearch-project/opensearch-api-specification/pull/731))
 
 ### Removed
 - Removed unsupported `_common.mapping:SourceField`'s `mode` field and associated `_common.mapping:SourceFieldMode` enum ([#652](https://github.com/opensearch-project/opensearch-api-specification/pull/652))
@@ -69,6 +70,8 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Fixed response schema for `/_render/template` and `/_render/template/{id}` ([#724](https://github.com/opensearch-project/opensearch-api-specification/pull/724))
 - Fixed data stream schema numeric property types ([#725](https://github.com/opensearch-project/opensearch-api-specification/pull/725))
 - Fixed snapshot status numeric property types ([#729](https://github.com/opensearch-project/opensearch-api-specification/pull/729))
+- Fixed request schema for `PATCH /_plugins/_security/api/nodesdn` ([#731](https://github.com/opensearch-project/opensearch-api-specification/pull/731))
+- Fixed response schema for `GET /_plugins/_security/api/nodesdn/{cluster_name}` ([#731](https://github.com/opensearch-project/opensearch-api-specification/pull/731))
 
 ### Changed
 - Changed `tasks._common:TaskInfo` and `tasks._common:TaskGroup` to be composed of a `tasks._common:TaskInfoBase` ([#683](https://github.com/opensearch-project/opensearch-api-specification/pull/683))

DEVELOPER_GUIDE.md

@@ -243,6 +243,8 @@ The dump-cluster-spec tool connects to an OpenSearch cluster which has the [open
 - `--opensearch-insecure`: Disable SSL/TLS certificate verification, defaults to performing verification.
 - `--opensearch-username <username>`: The username to authenticate with the cluster, defaults to `admin`, only used when `--opensearch-password` is set.
 - `--opensearch-password <password>`: The password to authenticate with the cluster, also settable via the `OPENSEARCH_PASSWORD` environment variable.
+- `--opensearch-cert <file>`: The OpenSSL certificate file, also settable via the `OPENSEARCH_CERT` environment variable.
+- `--opensearch-key <file>`: The OpenSSL certificate private key, also settable via the `OPENSEARCH_KEY` environment variable.
 - `--output <path>`: The path to write the dumped spec to, defaults to `<repository-root>/build/opensearch-openapi-CLUSTER.yaml`.
 
 **Example**

TESTING_GUIDE.md

@@ -5,6 +5,7 @@
       - [Prerequisites](#prerequisites)
       - [OpenSearch Cluster](#opensearch-cluster)
       - [Run Tests](#run-tests)
+    - [Running Spec Tests that Require an Admin Certificate](#running-spec-tests-that-require-an-admin-certificate)
     - [Running Spec Tests with Amazon OpenSearch](#running-spec-tests-with-amazon-opensearch)
     - [Common Errors](#common-errors)
       - [401 Unauthorized](#401-unauthorized)
@@ -76,6 +77,19 @@ Want to help with some missing tests? Choose from the remaining paths in the tes
 npm run test:spec -- --opensearch-insecure --coverage-report
 ```
 
+### Running Spec Tests that Require an Admin Certificate
+
+Some tests may require an admin certificate for authorization. The certificate can be provided wth `--opensearch-cert` and the key with `opensearch-key`.
+
+For example, run tests in [plugins/security](tests/plugins/security) as follows:
+```bash
+npm run test:spec--insecure -- \
+  --tests tests/plugins/security/api/nodesdn.yaml \
+  --opensearch-key tests/plugins/security/kirk-key.pem \
+  --opensearch-cert tests/plugins/security/kirk.pem \
+  --verbose
+```
+
 ### Running Spec Tests with Amazon OpenSearch
 
 Use an Amazon OpenSearch service instance.

spec/namespaces/security.yaml

@@ -754,6 +754,8 @@ paths:
       responses:
         '200':
           $ref: '#/components/responses/security.update_distinguished_name@200'
+        '201':
+          $ref: '#/components/responses/security.update_distinguished_name@201'
         '400':
           $ref: '#/components/responses/security.update_distinguished_name@400'
         '403':
@@ -1449,7 +1451,7 @@ components:
       content:
         application/json:
           schema:
-            $ref: '../schemas/security._common.yaml#/components/schemas/PatchOperation'
+            $ref: '../schemas/security._common.yaml#/components/schemas/PatchOperations'
     security.patch_distinguished_names:
       content:
         application/json:
@@ -1843,7 +1845,7 @@ components:
       content:
         application/json:
           schema:
-            $ref: '../schemas/security._common.yaml#/components/schemas/DistinguishedNames'
+            $ref: '../schemas/security._common.yaml#/components/schemas/DistinguishedNamesMap'
     security.get_distinguished_name@400:
       content:
         application/json:
@@ -2183,6 +2185,11 @@ components:
         application/json:
           schema:
             $ref: '../schemas/security._common.yaml#/components/schemas/Ok'
+    security.update_distinguished_name@201:
+      content:
+        application/json:
+          schema:
+            $ref: '../schemas/security._common.yaml#/components/schemas/Ok'
     security.update_distinguished_name@400:
       content:
         application/json:

spec/schemas/security._common.yaml

@@ -524,6 +524,11 @@ components:
           type: string
           description: Message returned as part of CREATED response.
 
+    PatchOperations:
+      type: array
+      items:
+        $ref: '#/components/schemas/PatchOperation'
+
     PatchOperation:
       type: object
       properties:

tests/plugins/security/.README.md

@@ -0,0 +1 @@
+The demo certs in this directory do get updated from [the security repo](https://github.com/opensearch-project/security/tree/main/bwc-test/src/test/resources/security). The source of truth is [here](https://github.com/opensearch-project/security/blob/main/src/main/java/org/opensearch/security/tools/democonfig/Certificates.java).

tests/plugins/security/.kirk-key.pem

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCVXDgEJQorgfXp
+gpY0TgF55bD2xuzxN5Dc9rDfgWxrsOvOloMpd7k6FR71bKWjJi1KptSmM/cDElky
+AWYKSfYWGiGxsQ+EQW+6kwCfEOHXQldn+0+JcWqP+osSPjtJfwRvRN5kRqP69MPo
+7U0N2kdqenqMWjmG1chDGLRSOEGU5HIBiDxsZtOcvMaJ8b1eaW0lvS+6gFQ80AvB
+GBkDDCOHHLtDXBylrZk2CQP8AzxNicIZ4B8G3CG3OHA8+nBtEtxZoIihrrkqlMt+
+b/5N8u8zB0Encew0kdrc4R/2wS//ahr6U+9Siq8T7WsUtGwKj3BJClg6OyDJRhlu
+y2gFnxoPAgMBAAECggEAP5TOycDkx+megAWVoHV2fmgvgZXkBrlzQwUG/VZQi7V4
+ZGzBMBVltdqI38wc5MtbK3TCgHANnnKgor9iq02Z4wXDwytPIiti/ycV9CDRKvv0
+TnD2hllQFjN/IUh5n4thHWbRTxmdM7cfcNgX3aZGkYbLBVVhOMtn4VwyYu/Mxy8j
+xClZT2xKOHkxqwmWPmdDTbAeZIbSv7RkIGfrKuQyUGUaWhrPslvYzFkYZ0umaDgQ
+OAthZew5Bz3OfUGOMPLH61SVPuJZh9zN1hTWOvT65WFWfsPd2yStI+WD/5PU1Doo
+1RyeHJO7s3ug8JPbtNJmaJwHe9nXBb/HXFdqb976yQKBgQDNYhpu+MYSYupaYqjs
+9YFmHQNKpNZqgZ4ceRFZ6cMJoqpI5dpEMqToFH7tpor72Lturct2U9nc2WR0HeEs
+/6tiptyMPTFEiMFb1opQlXF2ae7LeJllntDGN0Q6vxKnQV+7VMcXA0Y8F7tvGDy3
+qJu5lfvB1mNM2I6y/eMxjBuQhwKBgQC6K41DXMFro0UnoO879pOQYMydCErJRmjG
+/tZSy3Wj4KA/QJsDSViwGfvdPuHZRaG9WtxdL6kn0w1exM9Rb0bBKl36lvi7o7xv
+M+Lw9eyXMkww8/F5d7YYH77gIhGo+RITkKI3+5BxeBaUnrGvmHrpmpgRXWmINqr0
+0jsnN3u0OQKBgCf45vIgItSjQb8zonLz2SpZjTFy4XQ7I92gxnq8X0Q5z3B+o7tQ
+K/4rNwTju/sGFHyXAJlX+nfcK4vZ4OBUJjP+C8CTjEotX4yTNbo3S6zjMyGQqDI5
+9aIOUY4pb+TzeUFJX7If5gR+DfGyQubvvtcg1K3GHu9u2l8FwLj87sRzAoGAflQF
+RHuRiG+/AngTPnZAhc0Zq0kwLkpH2Rid6IrFZhGLy8AUL/O6aa0IGoaMDLpSWUJp
+nBY2S57MSM11/MVslrEgGmYNnI4r1K25xlaqV6K6ztEJv6n69327MS4NG8L/gCU5
+3pEm38hkUi8pVYU7in7rx4TCkrq94OkzWJYurAkCgYATQCL/rJLQAlJIGulp8s6h
+mQGwy8vIqMjAdHGLrCS35sVYBXG13knS52LJHvbVee39AbD5/LlWvjJGlQMzCLrw
+F7oILW5kXxhb8S73GWcuMbuQMFVHFONbZAZgn+C9FW4l7XyRdkrbR1MRZ2km8YMs
+/AHmo368d4PSNRMMzLHw8Q==
+-----END PRIVATE KEY-----

tests/plugins/security/.kirk.pem

@@ -0,0 +1,27 @@
+-----BEGIN CERTIFICATE-----
+MIIEmDCCA4CgAwIBAgIUaYSlET3nzsotWTrWueVPPh10yLcwDQYJKoZIhvcNAQEL
+BQAwgY8xEzARBgoJkiaJk/IsZAEZFgNjb20xFzAVBgoJkiaJk/IsZAEZFgdleGFt
+cGxlMRkwFwYDVQQKDBBFeGFtcGxlIENvbSBJbmMuMSEwHwYDVQQLDBhFeGFtcGxl
+IENvbSBJbmMuIFJvb3QgQ0ExITAfBgNVBAMMGEV4YW1wbGUgQ29tIEluYy4gUm9v
+dCBDQTAeFw0yNDAyMjAxNzA0MjRaFw0zNDAyMTcxNzA0MjRaME0xCzAJBgNVBAYT
+AmRlMQ0wCwYDVQQHDAR0ZXN0MQ8wDQYDVQQKDAZjbGllbnQxDzANBgNVBAsMBmNs
+aWVudDENMAsGA1UEAwwEa2lyazCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC
+ggEBAJVcOAQlCiuB9emCljROAXnlsPbG7PE3kNz2sN+BbGuw686Wgyl3uToVHvVs
+paMmLUqm1KYz9wMSWTIBZgpJ9hYaIbGxD4RBb7qTAJ8Q4ddCV2f7T4lxao/6ixI+
+O0l/BG9E3mRGo/r0w+jtTQ3aR2p6eoxaOYbVyEMYtFI4QZTkcgGIPGxm05y8xonx
+vV5pbSW9L7qAVDzQC8EYGQMMI4ccu0NcHKWtmTYJA/wDPE2JwhngHwbcIbc4cDz6
+cG0S3FmgiKGuuSqUy35v/k3y7zMHQSdx7DSR2tzhH/bBL/9qGvpT71KKrxPtaxS0
+bAqPcEkKWDo7IMlGGW7LaAWfGg8CAwEAAaOCASswggEnMAwGA1UdEwEB/wQCMAAw
+DgYDVR0PAQH/BAQDAgXgMBYGA1UdJQEB/wQMMAoGCCsGAQUFBwMCMB0GA1UdDgQW
+BBSjMS8tgguX/V7KSGLoGg7K6XMzIDCBzwYDVR0jBIHHMIHEgBQXh9+gWutmEqfV
+0Pi6EkU8tysAnKGBlaSBkjCBjzETMBEGCgmSJomT8ixkARkWA2NvbTEXMBUGCgmS
+JomT8ixkARkWB2V4YW1wbGUxGTAXBgNVBAoMEEV4YW1wbGUgQ29tIEluYy4xITAf
+BgNVBAsMGEV4YW1wbGUgQ29tIEluYy4gUm9vdCBDQTEhMB8GA1UEAwwYRXhhbXBs
+ZSBDb20gSW5jLiBSb290IENBghQNZAmZZn3EFOxBR4630XlhI+mo4jANBgkqhkiG
+9w0BAQsFAAOCAQEACEUPPE66/Ot3vZqRGpjDjPHAdtOq+ebaglQhvYcnDw8LOZm8
+Gbh9M88CiO6UxC8ipQLTPh2yyeWArkpJzJK/Pi1eoF1XLiAa0sQ/RaJfQWPm9dvl
+1ZQeK5vfD4147b3iBobwEV+CR04SKow0YeEEzAJvzr8YdKI6jqr+2GjjVqzxvRBy
+KRVHWCFiR7bZhHGLq3br8hSu0hwjb3oGa1ZI8dui6ujyZt6nm6BoEkau3G/6+zq9
+E6vX3+8Fj4HKCAL6i0SwfGmEpTNp5WUhqibK/fMhhmMT4Mx6MxkT+OFnIjdUU0S/
+e3kgnG8qjficUr38CyEli1U0M7koIXUZI7r+LQ==
+-----END CERTIFICATE-----

tests/plugins/security/api/nodesdn.yaml

@@ -0,0 +1,64 @@
+$schema: ../../../../json_schemas/test_story.schema.yaml
+
+description: Test dynamic distinguished names.
+epilogues:
+  - path: /_plugins/_security/api/nodesdn/opensearch-cluster
+    method: DELETE
+    status: [200,404]
+chapters:
+  - synopsis: Add distinguished names.
+    path: /_plugins/_security/api/nodesdn/{cluster_name}
+    method: PUT
+    parameters:
+      cluster_name: opensearch-cluster
+    request:
+      payload:
+        nodes_dn:
+          - CN=cluster.example.com
+    response:
+      status: 201
+      payload:
+        status: CREATED
+  - synopsis: Retrieve distinguished names.
+    path: /_plugins/_security/api/nodesdn
+    method: GET
+  - synopsis: Update distinguished names.
+    path: /_plugins/_security/api/nodesdn
+    method: PATCH
+    request:
+      payload:
+        - op: replace
+          path: /opensearch-cluster/nodes_dn/0
+          value: CN=Christopher Nolan,CN=admin,DC=movies,DC=opensearch,DC=org
+    response:
+      status: 200
+      payload:
+        status: OK
+  - synopsis: Update distinguished names (with cluster name).
+    path: /_plugins/_security/api/nodesdn/{cluster_name}
+    method: PATCH
+    parameters:
+      cluster_name: opensearch-cluster
+    request:
+      payload:
+        - op: replace
+          path: /nodes_dn/0
+          value: CN=Quentin Tarantino,CN=admin,DC=movies,DC=opensearch,DC=org
+    response:
+      status: 200
+      payload:
+        status: OK
+  - synopsis: Retrieve distinguished names (with cluster name).
+    path: /_plugins/_security/api/nodesdn/{cluster_name}
+    method: GET
+    parameters:
+      cluster_name: opensearch-cluster
+  - synopsis: Delete all distinguished names.
+    path: /_plugins/_security/api/nodesdn/{cluster_name}
+    method: DELETE
+    parameters:
+      cluster_name: opensearch-cluster
+    response:
+      status: 200
+      payload:
+        status: OK

tests/plugins/security/docker-compose.yml

@@ -0,0 +1,13 @@
+version: '3'
+
+services:
+  opensearch-cluster:
+    image: ${OPENSEARCH_DOCKER_HUB_PROJECT:-opensearchproject}/opensearch:${OPENSEARCH_VERSION:-latest}${OPENSEARCH_DOCKER_REF}
+    ports:
+      - 9200:9200
+      - 9600:9600
+    environment:
+      - OPENSEARCH_INITIAL_ADMIN_PASSWORD=${OPENSEARCH_PASSWORD:-myStrongPassword123!}
+      - OPENSEARCH_JAVA_OPTS=${OPENSEARCH_JAVA_OPTS}
+      - discovery.type=single-node
+      - plugins.security.nodes_dn_dynamic_config_enabled=true

tools/src/OpenSearchHttpClient.ts

@@ -10,6 +10,7 @@
 import { Option } from '@commander-js/extra-typings'
 import axios, { type AxiosInstance, type AxiosRequestConfig, type AxiosResponse, type ResponseType } from 'axios'
 import * as https from 'node:https'
+import fs from 'fs'
 import { sleep } from './helpers'
 import { Logger } from './Logger'
 import { aws4Interceptor } from 'aws4-axios'
@@ -29,6 +30,12 @@ export const OPENSEARCH_USERNAME_OPTION = new Option('--opensearch-username <use
 export const OPENSEARCH_PASSWORD_OPTION = new Option('--opensearch-password <password>', 'password to use when authenticating with OpenSearch')
   .env('OPENSEARCH_PASSWORD')
 
+export const OPENSEARCH_CERT_OPTION = new Option('--opensearch-cert <cert>', 'client certificate file to use when authenticating with OpenSearch')
+  .env('OPENSEARCH_CERT')
+
+export const OPENSEARCH_KEY_OPTION = new Option('--opensearch-key <cert>', 'client certificate private key file name to use when authenticating with OpenSearch')
+  .env('OPENSEARCH_KEY')
+
 export const OPENSEARCH_INSECURE_OPTION = new Option('--opensearch-insecure', 'disable SSL/TLS certificate verification when connecting to OpenSearch')
   .default(DEFAULT_INSECURE)
 
@@ -65,6 +72,8 @@ export interface AwsAuth {
 export interface OpenSearchHttpClientOptions {
   url?: string
   insecure?: boolean
+  cert?: string,
+  key?: string,
   responseType?: ResponseType
   logger?: Logger,
   basic_auth?: BasicAuth
@@ -77,6 +86,8 @@ export type OpenSearchHttpClientCliOptions = {
   opensearchUsername?: string
   opensearchPassword?: string
   opensearchInsecure?: boolean
+  opensearchCert?: string,
+  opensearchKey?: string,
   awsAccessKeyId?: string
   awsSecretAccessKey?: string
   awsSessionToken?: string
@@ -90,6 +101,8 @@ export function get_opensearch_opts_from_cli (opts: OpenSearchHttpClientCliOptio
   return {
     url: opts.opensearchUrl,
     insecure: opts.opensearchInsecure,
+    cert: opts.opensearchCert,
+    key: opts.opensearchKey,
     basic_auth: opts.opensearchUsername !== undefined && opts.opensearchPassword !== undefined ? {
       username: opts.opensearchUsername,
       password: opts.opensearchPassword
@@ -164,7 +177,11 @@ export class OpenSearchHttpClient {
 
     this._axios = axios.create({
       baseURL: opts?.url ?? DEFAULT_URL,
-      httpsAgent: new https.Agent({ rejectUnauthorized: !(opts?.insecure ?? DEFAULT_INSECURE) }),
+      httpsAgent: new https.Agent({
+        rejectUnauthorized: !(opts?.insecure ?? DEFAULT_INSECURE),
+        cert: opts?.cert !== undefined && opts?.cert !== '' ? fs.readFileSync(opts?.cert) : undefined,
+        key: opts?.key !== undefined && opts?.key !== '' ? fs.readFileSync(opts?.key) : undefined,
+      }),
       responseType: opts?.responseType,
     })
 

tools/src/dump-cluster-spec/dump-cluster-spec.ts

@@ -13,7 +13,9 @@ import * as process from 'node:process'
 import { write_yaml } from '../helpers'
 import {
   get_opensearch_opts_from_cli,
+  OPENSEARCH_CERT_OPTION,
   OPENSEARCH_INSECURE_OPTION,
+  OPENSEARCH_KEY_OPTION,
   OPENSEARCH_PASSWORD_OPTION,
   OPENSEARCH_URL_OPTION,
   OPENSEARCH_USERNAME_OPTION, OpenSearchHttpClient,
@@ -42,6 +44,8 @@ const command = new Command()
   .addOption(OPENSEARCH_USERNAME_OPTION)
   .addOption(OPENSEARCH_PASSWORD_OPTION)
   .addOption(OPENSEARCH_INSECURE_OPTION)
+  .addOption(OPENSEARCH_CERT_OPTION)
+  .addOption(OPENSEARCH_KEY_OPTION)
   .addOption(new Option('--output <path>', 'path to the output file').default(resolve(__dirname, '../../../build/opensearch-openapi-CLUSTER.yaml')))
   .allowExcessArguments(false)
   .parse()

tools/src/tester/test.ts

@@ -17,7 +17,9 @@ import {
   AWS_SERVICE_OPTION,
   AWS_SESSION_TOKEN_OPTION,
   get_opensearch_opts_from_cli,
+  OPENSEARCH_CERT_OPTION,
   OPENSEARCH_INSECURE_OPTION,
+  OPENSEARCH_KEY_OPTION,
   OPENSEARCH_PASSWORD_OPTION,
   OPENSEARCH_URL_OPTION,
   OPENSEARCH_USERNAME_OPTION,
@@ -54,6 +56,8 @@ const command = new Command()
   .addOption(OPENSEARCH_USERNAME_OPTION)
   .addOption(OPENSEARCH_PASSWORD_OPTION)
   .addOption(OPENSEARCH_INSECURE_OPTION)
+  .addOption(OPENSEARCH_CERT_OPTION)
+  .addOption(OPENSEARCH_KEY_OPTION)
   .addOption(AWS_ACCESS_KEY_ID_OPTION)
   .addOption(AWS_SECRET_ACCESS_KEY_OPTION)
   .addOption(AWS_SESSION_TOKEN_OPTION)