Split up OpenApiVersionExtractor from OpenApiMerger.

Signed-off-by: dblock <[email protected]>

tools/src/linter/SchemasValidator.ts

@@ -32,7 +32,7 @@ export default class SchemasValidator {
   }
 
   validate (): ValidationError[] {
-    const merger = new OpenApiMerger(this.root_folder, undefined, new Logger(LogLevel.error))
+    const merger = new OpenApiMerger(this.root_folder, new Logger(LogLevel.error))
     this.spec = merger.spec().components as Record<string, any>
     const named_schemas_errors = this.validate_named_schemas()
     if (named_schemas_errors.length > 0) return named_schemas_errors

tools/src/merger/OpenApiMerger.ts

@@ -9,29 +9,26 @@
 
 import { type OpenAPIV3 } from 'openapi-types'
 import fs from 'fs'
-import _, { isEmpty } from 'lodash'
-import { delete_matching_keys, read_yaml, write_yaml } from '../helpers'
+import _ from 'lodash'
+import { read_yaml, write_yaml } from '../helpers'
 import SupersededOpsGenerator from './SupersededOpsGenerator'
 import GlobalParamsGenerator from './GlobalParamsGenerator'
 import { Logger } from '../Logger'
-import * as semver from 'semver'
 
 // Create a single-file OpenAPI spec from multiple files for OpenAPI validation and programmatic consumption
 export default class OpenApiMerger {
   root_folder: string
   logger: Logger
-  target_version?: string
 
   protected _spec: Record<string, any>
   protected _merged: boolean = false
 
   paths: Record<string, Record<string, OpenAPIV3.PathItemObject>> = {} // namespace -> path -> path_item_object
   schemas: Record<string, Record<string, OpenAPIV3.SchemaObject>> = {} // category -> schema -> schema_object
 
-  constructor (root_folder: string, target_version?: string, logger: Logger = new Logger()) {
+  constructor (root_folder: string, logger: Logger = new Logger()) {
     this.logger = logger
     this.root_folder = fs.realpathSync(root_folder)
-    this.target_version = target_version === undefined ? undefined : semver.coerce(target_version)?.toString()
     this._spec = {
       openapi: '3.1.0',
       info: read_yaml(`${this.root_folder}/_info.yaml`, true),
@@ -76,54 +73,6 @@ export default class OpenApiMerger {
       this._spec.components.responses = { ...this._spec.components.responses, ...spec.components.responses }
       this._spec.components.requestBodies = { ...this._spec.components.requestBodies, ...spec.components.requestBodies }
     })
-
-    this.#remove_refs_per_semver()
-  }
-
-  // Remove any refs that are x-version-added/removed incompatible with the target server version.
-  #remove_refs_per_semver() : void {
-    this.#remove_keys_not_matching_semver(this._spec.paths)
-
-    // parameters
-    const removed_params = this.#remove_keys_not_matching_semver(this._spec.components.parameters)
-    const removed_parameter_refs = _.map(removed_params, (ref) => `#/components/parameters/${ref}`)
-    Object.entries(this._spec.paths as Document).forEach(([_path, path_item]) => {
-      Object.entries(path_item as Document).forEach(([_method, method_item]) => {
-        method_item.parameters = _.filter(method_item.parameters, (param) => !_.includes(removed_parameter_refs, param.$ref))
-      })
-    })
-
-    // responses
-    const removed_responses = this.#remove_keys_not_matching_semver(this._spec.components.responses)
-    const removed_response_refs = _.map(removed_responses, (ref) => `#/components/responses/${ref}`)
-    Object.entries(this._spec.paths as Document).forEach(([_path, path_item]) => {
-      Object.entries(path_item as Document).forEach(([_method, method_item]) => {
-        method_item.responses = _.omitBy(method_item.responses, (param) => _.includes(removed_response_refs, param.$ref))
-      })
-    })
-
-    this._spec.paths = _.omitBy(this._spec.paths, isEmpty)
-  }
-
-  #exclude_per_semver(obj: any): boolean {
-    if (this.target_version === undefined) return false
-
-    const x_version_added = semver.coerce(obj['x-version-added'] as string)
-    const x_version_removed = semver.coerce(obj['x-version-removed'] as string)
-
-    if (x_version_added && !semver.satisfies(this.target_version, `>=${x_version_added?.toString()}`)) {
-      return true
-    } else if (x_version_removed && !semver.satisfies(this.target_version, `<${x_version_removed?.toString()}`)) {
-      return true
-    }
-
-    return false
-  }
-
-  // Remove any elements that are x-version-added/removed incompatible with the target server version.
-  #remove_keys_not_matching_semver(obj: any): string[] {
-    if (this.target_version === undefined) return []
-    return delete_matching_keys(obj, this.#exclude_per_semver.bind(this))
   }
 
   // Redirect schema references in namespace files to local references in single-file spec.

tools/src/merger/OpenApiVersionExtractor.ts

@@ -0,0 +1,102 @@
+/*
+* Copyright OpenSearch Contributors
+* SPDX-License-Identifier: Apache-2.0
+*
+* The OpenSearch Contributors require contributions made to
+* this file be licensed under the Apache-2.0 license or a
+* compatible open source license.
+*/
+
+import { Logger } from '../Logger'
+import { delete_matching_keys, write_yaml } from '../helpers'
+import _, { isEmpty } from 'lodash'
+import { type OpenAPIV3 } from 'openapi-types'
+import semver from 'semver'
+
+// Extract a versioned API
+export default class OpenApiVersionExtractor {
+  private _spec?: Record<string, any>
+  private _source_spec: OpenAPIV3.Document
+  private _target_version?: string
+  private _logger: Logger
+
+  constructor(source_spec: OpenAPIV3.Document, target_version?: string, logger: Logger = new Logger()) {
+    this._source_spec = source_spec
+    this._target_version = target_version !== undefined ? semver.coerce(target_version)?.toString() : undefined
+    this._logger = logger
+    this._spec = undefined
+  }
+
+  extract(): OpenAPIV3.Document {
+    if (this._spec) return this._spec as OpenAPIV3.Document
+    if (this._target_version !== undefined) {
+      this.#extract()
+    } else {
+      this._spec = this._source_spec
+    }
+    return this._spec as OpenAPIV3.Document
+  }
+
+  write_to(output_path: string): OpenApiVersionExtractor {
+    this._logger.info(`Writing ${output_path} ...`)
+    write_yaml(output_path, this.extract())
+    return this
+  }
+
+  // Remove any refs that are x-version-added/removed incompatible with the target server version.
+  #extract() : void {
+    this._logger.info(`Extracting version ${this._target_version} ...`)
+
+    this._spec = _.cloneDeep(this._source_spec)
+
+    this._spec.components = this._spec.components ?? {
+      parameters: {},
+      requestBodies: {},
+      responses: {},
+      schemas: {}
+    }
+
+    this.#remove_keys_not_matching_semver(this._spec.paths)
+
+    // parameters
+    const removed_params = this.#remove_keys_not_matching_semver(this._spec.components.parameters)
+    const removed_parameter_refs = _.map(removed_params, (ref) => `#/components/parameters/${ref}`)
+    Object.entries(this._spec.paths as Document).forEach(([_path, path_item]) => {
+      Object.entries(path_item as Document).forEach(([_method, method_item]) => {
+        method_item.parameters = _.filter(method_item.parameters, (param) => !_.includes(removed_parameter_refs, param.$ref))
+      })
+    })
+
+    // responses
+    const removed_responses = this.#remove_keys_not_matching_semver(this._spec.components.responses)
+    const removed_response_refs = _.map(removed_responses, (ref) => `#/components/responses/${ref}`)
+    Object.entries(this._spec.paths as Document).forEach(([_path, path_item]) => {
+      Object.entries(path_item as Document).forEach(([_method, method_item]) => {
+        method_item.responses = _.omitBy(method_item.responses, (param) => _.includes(removed_response_refs, param.$ref))
+      })
+    })
+
+    this._spec.paths = _.omitBy(this._spec.paths, isEmpty)
+  }
+
+  #exclude_per_semver(obj: any): boolean {
+    if (this._target_version === undefined) return false
+
+    const x_version_added = semver.coerce(obj['x-version-added'] as string)
+    const x_version_removed = semver.coerce(obj['x-version-removed'] as string)
+
+    if (x_version_added && !semver.satisfies(this._target_version, `>=${x_version_added?.toString()}`)) {
+      return true
+    } else if (x_version_removed && !semver.satisfies(this._target_version, `<${x_version_removed?.toString()}`)) {
+      return true
+    }
+
+    return false
+  }
+
+  // Remove any elements that are x-version-added/removed incompatible with the target server version.
+  #remove_keys_not_matching_semver(obj: any): string[] {
+    if (this._target_version === undefined) return []
+    return delete_matching_keys(obj, this.#exclude_per_semver.bind(this))
+  }
+}

tools/src/merger/merge.ts

@@ -8,9 +8,10 @@
 */
 
 import { Command, Option } from '@commander-js/extra-typings'
-import OpenApiMerger from './OpenApiMerger'
-import { resolve } from 'path'
 import { Logger, LogLevel } from '../Logger'
+import { resolve } from 'path'
+import OpenApiMerger from './OpenApiMerger'
+import OpenApiVersionExtractor from './OpenApiVersionExtractor'
 
 const command = new Command()
   .description('Merges the multi-file OpenSearch spec into a single file for programmatic use.')
@@ -23,7 +24,13 @@ const command = new Command()
 
 const opts = command.opts()
 const logger = new Logger(opts.verbose ? LogLevel.info : LogLevel.warn)
-const merger = new OpenApiMerger(opts.source, opts.opensearchVersion, logger)
-logger.log(`Merging ${opts.source} into ${opts.output} (${opts.opensearchVersion}) ...`)
-merger.write_to(opts.output)
+const merger = new OpenApiMerger(opts.source, logger)
+if (opts.opensearchVersion === undefined) {
+  logger.log(`Merging ${opts.source} into ${opts.output} ...`)
+  merger.write_to(opts.output)
+} else {
+  logger.log(`Merging ${opts.source} into ${opts.output} (${opts.opensearchVersion}) ...`)
+  const extractor = new OpenApiVersionExtractor(merger.spec(), opts.opensearchVersion)
+  extractor.write_to(opts.output)
+}
 logger.log('Done.')

tools/src/tester/MergedOpenApiSpec.ts

@@ -13,6 +13,7 @@ import { determine_possible_schema_types, HTTP_METHODS, SpecificationContext } f
 import { SchemaVisitor } from '../_utils/SpecificationVisitor';
 import OpenApiMerger from '../merger/OpenApiMerger';
 import _ from 'lodash';
+import OpenApiVersionExtractor from '../merger/OpenApiVersionExtractor';
 
 // An augmented spec with additionalProperties: false.
 export default class MergedOpenApiSpec {
@@ -30,8 +31,12 @@ export default class MergedOpenApiSpec {
 
   spec (): OpenAPIV3.Document {
     if (this._spec) return this._spec
-    const merger = new OpenApiMerger(this.file_path, this.target_version, this.logger)
-    const spec = merger.spec()
+    const merger = new OpenApiMerger(this.file_path, this.logger)
+    var spec = merger.spec()
+    if (this.target_version !== undefined) {
+      const version_extractor = new OpenApiVersionExtractor(spec, this.target_version)
+      spec = version_extractor.extract()
+    }
     const ctx = new SpecificationContext(this.file_path)
     this.inject_additional_properties(ctx, spec)
     this._spec = spec

tools/tests/merger/OpenApiMerger.test.ts

@@ -45,31 +45,9 @@ describe('OpenApiMerger', () => {
 
       test('writes a spec', () => {
         merger.write_to(filename)
-        expect(fs.readFileSync('./tools/tests/merger/fixtures/expected_2.0.yaml', 'utf8'))
+        expect(fs.readFileSync('./tools/tests/merger/fixtures/merger/expected.yaml', 'utf8'))
           .toEqual(fs.readFileSync(filename, 'utf8'))
       })
     })
   })
-
-  describe('1.3', () => {
-    var temp: tmp.DirResult
-    var filename: string
-
-    beforeEach(() => {
-      merger = new OpenApiMerger('./tools/tests/merger/fixtures/spec/', '1.3')
-      temp = tmp.dirSync()
-      filename = `${temp.name}/opensearch-openapi.yaml`
-    })
-
-    afterEach(() => {
-      fs.unlinkSync(filename)
-      temp.removeCallback()
-    })
-
-    test('writes a spec', () => {
-      merger.write_to(filename)
-      expect(fs.readFileSync('./tools/tests/merger/fixtures/expected_1.3.yaml', 'utf8'))
-        .toEqual(fs.readFileSync(filename, 'utf8'))
-    })
-  })
 })

tools/tests/merger/OpenApiVersionExtractor.test.ts

@@ -0,0 +1,105 @@
+/*
+* Copyright OpenSearch Contributors
+* SPDX-License-Identifier: Apache-2.0
+*
+* The OpenSearch Contributors require contributions made to
+* this file be licensed under the Apache-2.0 license or a
+* compatible open source license.
+*/
+
+import _ from 'lodash'
+import OpenApiMerger from 'merger/OpenApiMerger'
+import OpenApiVersionExtractor from 'merger/OpenApiVersionExtractor'
+import fs from 'fs'
+import tmp from 'tmp'
+
+describe('extract() from a merged API spec', () => {
+  const merger = new OpenApiMerger('tools/tests/tester/fixtures/specs/complete')
+
+  describe('defaults', () => {
+    const extractor = new OpenApiVersionExtractor(merger.spec())
+
+    test('has all responses', () => {
+      const spec = extractor.extract()
+
+      expect(_.keys(spec.paths['/index']?.get?.responses)).toEqual([
+        '200', '201', '404', '500', 'added-2.0', 'removed-2.0', 'added-1.3-removed-2.0', 'added-2.1'
+      ])
+    })
+
+    describe('write_to()', () => {
+      var temp: tmp.DirResult
+      var filename: string
+
+      beforeEach(() => {
+        temp = tmp.dirSync()
+        filename = `${temp.name}/opensearch-openapi.yaml`
+      })
+
+      afterEach(() => {
+        fs.unlinkSync(filename)
+        temp.removeCallback()
+      })
+
+      test('writes a spec', () => {
+        extractor.write_to(filename)
+        expect(fs.readFileSync('./tools/tests/merger/fixtures/extractor/expected_2.0.yaml', 'utf8'))
+          .toEqual(fs.readFileSync(filename, 'utf8'))
+      })
+    })
+  })
+
+  describe('1.3', () => {
+    const extractor = new OpenApiVersionExtractor(merger.spec(), '1.3')
+
+    describe('write_to', () => {
+      var temp: tmp.DirResult
+      var filename: string
+
+      beforeEach(() => {
+        temp = tmp.dirSync()
+        filename = `${temp.name}/opensearch-openapi.yaml`
+      })
+
+      afterEach(() => {
+        fs.unlinkSync(filename)
+        temp.removeCallback()
+      })
+
+      test('writes a spec', () => {
+        extractor.write_to(filename)
+        expect(fs.readFileSync('./tools/tests/merger/fixtures/extractor/expected_1.3.yaml', 'utf8'))
+          .toEqual(fs.readFileSync(filename, 'utf8'))
+      })
+    })
+
+    test('has matching responses', () => {
+      const spec = extractor.extract()
+      expect(_.keys(spec.paths['/index']?.get?.responses)).toEqual([
+        '200', '201', '404', '500', 'removed-2.0', 'added-1.3-removed-2.0'
+      ])
+    })
+
+    describe('2.0', () => {
+      const extractor = new OpenApiVersionExtractor(merger.spec(), '2.0')
+
+      test('has matching responses', () => {
+        const spec = extractor.extract()
+        expect(_.keys(spec.paths['/index']?.get?.responses)).toEqual([
+          '200', '201', '404', '500', 'added-2.0'
+        ])
+      })
+    })
+
+    describe('2.1', () => {
+      const extractor = new OpenApiVersionExtractor(merger.spec(), '2.1')
+
+      test('has matching responses', () => {
+        const spec = extractor.extract()
+        expect(_.keys(spec.paths['/index']?.get?.responses)).toEqual([
+          '200', '201', '404', '500', 'added-2.0', 'added-2.1'
+        ])
+      })
+    })
+  })
+})