diff --git a/package.json b/package.json index 974ef1145..0aab88335 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ "merge": "ts-node tools/src/merger/merge.ts", "lint:spec": "ts-node tools/src/linter/lint.ts", "lint": "eslint .", + "lint--fix": "eslint . --fix", "test": "jest" }, "dependencies": { diff --git a/spec/schemas/security._common.yaml b/spec/schemas/security._common.yaml index c7452bb2f..5d1bab264 100644 --- a/spec/schemas/security._common.yaml +++ b/spec/schemas/security._common.yaml @@ -422,15 +422,14 @@ components: description: type: string description: Contains the description supplied by the user to describe the token. - required: true service: type: string description: A name of the service if generating a token for that service. - required: false duration: type: string description: Value in seconds. - required: optional + required: + - description Ok: type: object diff --git a/tools/src/linter/SchemasValidator.ts b/tools/src/linter/SchemasValidator.ts new file mode 100644 index 000000000..8042fb556 --- /dev/null +++ b/tools/src/linter/SchemasValidator.ts @@ -0,0 +1,100 @@ +import AJV from 'ajv' +import addFormats from 'ajv-formats' +import OpenApiMerger from '../merger/OpenApiMerger' +import { type ValidationError } from '../types' + +export default class SchemasValidator { + root_folder: string + spec: Record = {} + ajv: AJV + + constructor (root_folder: string) { + this.root_folder = root_folder + this.ajv = new AJV() + addFormats(this.ajv) + } + + validate (): ValidationError[] { + this.spec = new OpenApiMerger(this.root_folder, true).merge().components as Record + const named_schemas_errors = this.validate_named_schemas() + if (named_schemas_errors.length > 0) return named_schemas_errors + return [ + ...this.validate_parameter_schemas(), + ...this.validate_request_body_schemas(), + ...this.validate_response_schemas() + ] + } + + validate_named_schemas (): ValidationError[] { + return Object.entries(this.spec.schemas as Record).map(([key, _schema]) => { + const schema = _schema as Record + const error = this.validate_schema(schema) + if (error == null) { + this.ajv.addSchema(schema, `#/components/schemas/${key}`) + return null + } + + const file = `schemas/${key.split(':')[0]}.yaml` + const location = `#/components/schemas/${key.split(':')[1]}` + return this.error(file, location, error) + }).filter((error) => error != null) as ValidationError[] + } + + validate_parameter_schemas (): ValidationError[] { + return Object.entries(this.spec.parameters as Record).map(([key, param]) => { + const error = this.validate_schema(param.schema as Record) + if (error == null) return + + const namespace = this.group_to_namespace(key.split('::')[0]) + const file = namespace === '_global' ? '_global_parameters.yaml' : `namespaces/${namespace}.yaml` + const location = namespace === '_global' ? param.name as string : `#/components/parameters/${key}` + return this.error(file, location, error) + }).filter((error) => error != null) as ValidationError[] + } + + validate_request_body_schemas (): ValidationError[] { + return Object.entries(this.spec.requestBodies as Record).flatMap(([namespace, body]) => { + const file = `namespaces/${namespace}.yaml` + const location = `#/components/requestBodies/${namespace}` + return this.validate_content_schemas(file, location, body.content as Record) + }) + } + + validate_response_schemas (): ValidationError[] { + return Object.entries(this.spec.responses as Record).flatMap(([key, response]) => { + const namespace = this.group_to_namespace(key.split('@')[0]) + const file = `namespaces/${namespace}.yaml` + const location = `#/components/responses/${key}` + const content = response.content as Record + return this.validate_content_schemas(file, location, content) + }) + } + + validate_content_schemas (file: string, location: string, content: Record | undefined): ValidationError[] { + return Object.entries(content ?? {}).map(([media_type, value]) => { + const schema = value.schema as Record + const error = this.validate_schema(schema) + if (error != null) return this.error(file, `${location}/content/${media_type}`, error) + }).filter(e => e != null) as ValidationError[] + } + + validate_schema (schema: Record): Error | undefined { + if (schema == null || schema.$ref != null) return + try { + // eslint-disable-next-line @typescript-eslint/no-floating-promises + this.ajv.validateSchema(schema, true) + } catch (error) { + return error as Error + } + } + + group_to_namespace (group: string): string { + if (group === '_global') return '_global' + const [, namespace] = group.split('.').reverse() + return namespace ?? '_core' + } + + error (file: string, location: string, error: Error): ValidationError { + return { file, location, message: error.message } + } +} diff --git a/tools/src/linter/SpecValidator.ts b/tools/src/linter/SpecValidator.ts index 12da4175c..51ccf2a27 100644 --- a/tools/src/linter/SpecValidator.ts +++ b/tools/src/linter/SpecValidator.ts @@ -5,12 +5,14 @@ import SchemaRefsValidator from './SchemaRefsValidator' import SupersededOperationsFile from './components/SupersededOperationsFile' import InfoFile from './components/InfoFile' import InlineObjectSchemaValidator from './InlineObjectSchemaValidator' +import SchemasValidator from './SchemasValidator' export default class SpecValidator { superseded_ops_file: SupersededOperationsFile info_file: InfoFile namespaces_folder: NamespacesFolder schemas_folder: SchemasFolder + schemas_validator: SchemasValidator schema_refs_validator: SchemaRefsValidator inline_object_schema_validator: InlineObjectSchemaValidator @@ -19,6 +21,7 @@ export default class SpecValidator { this.info_file = new InfoFile(`${root_folder}/_info.yaml`) this.namespaces_folder = new NamespacesFolder(`${root_folder}/namespaces`) this.schemas_folder = new SchemasFolder(`${root_folder}/schemas`) + this.schemas_validator = new SchemasValidator(root_folder) this.schema_refs_validator = new SchemaRefsValidator(this.namespaces_folder, this.schemas_folder) this.inline_object_schema_validator = new InlineObjectSchemaValidator(this.namespaces_folder, this.schemas_folder) } @@ -34,7 +37,8 @@ export default class SpecValidator { ...this.schema_refs_validator.validate(), ...this.superseded_ops_file.validate(), ...this.info_file.validate(), - ...this.inline_object_schema_validator.validate() + ...this.inline_object_schema_validator.validate(), + ...this.schemas_validator.validate() ] } } diff --git a/tools/src/merger/OpenApiMerger.ts b/tools/src/merger/OpenApiMerger.ts index c2d8bec6f..dfa73bfda 100644 --- a/tools/src/merger/OpenApiMerger.ts +++ b/tools/src/merger/OpenApiMerger.ts @@ -9,11 +9,13 @@ import GlobalParamsGenerator from './GlobalParamsGenerator' export default class OpenApiMerger { root_folder: string spec: Record + silent: boolean paths: Record> = {} // namespace -> path -> path_item_object schemas: Record> = {} // category -> schema -> schema_object - constructor (root_folder: string) { + constructor (root_folder: string, silent: boolean = false) { + this.silent = silent this.root_folder = fs.realpathSync(root_folder) this.spec = { openapi: '3.1.0', @@ -117,6 +119,6 @@ export default class OpenApiMerger { // Generate superseded operations from _superseded_operations.yaml file. #generate_superseded_ops (): void { const gen = new SupersededOpsGenerator(this.root_folder) - gen.generate(this.spec) + gen.generate(this.spec, this.silent) } } diff --git a/tools/src/merger/SupersededOpsGenerator.ts b/tools/src/merger/SupersededOpsGenerator.ts index b673aa3d1..19dfa3b42 100644 --- a/tools/src/merger/SupersededOpsGenerator.ts +++ b/tools/src/merger/SupersededOpsGenerator.ts @@ -11,14 +11,14 @@ export default class SupersededOpsGenerator { delete this.superseded_ops.$schema } - generate (spec: Record): void { + generate (spec: Record, silent: boolean): void { for (const [path, { superseded_by, operations }] of _.entries(this.superseded_ops)) { const regex = this.path_to_regex(superseded_by) const operation_keys = operations.map(op => op.toLowerCase()) const superseded_path = this.copy_params(superseded_by, path) const path_entry = _.entries(spec.paths as Document).find(([path, _]) => regex.test(path)) - if (!path_entry) console.log(`Path not found: ${superseded_by}`) - else spec.paths[superseded_path] = this.path_object(path_entry[1], operation_keys) + if (path_entry != null) spec.paths[superseded_path] = this.path_object(path_entry[1], operation_keys) + else if (!silent) console.log(`Path not found: ${superseded_by}`) } } diff --git a/tools/src/merger/merge.ts b/tools/src/merger/merge.ts index c38f29c97..a8b33526a 100644 --- a/tools/src/merger/merge.ts +++ b/tools/src/merger/merge.ts @@ -5,7 +5,7 @@ import { resolve } from 'path' const command = new Command() .description('Merges the multi-file OpenSearch spec into a single file for programmatic use.') .addOption(new Option('-s, --source ', 'path to the root folder of the multi-file spec').default(resolve(__dirname, '../../../spec'))) - .addOption(new Option('-o, --output ', 'output file name').default(resolve(__dirname, '../../../build/opensearch-openapi.yaml'))) + .addOption(new Option('-o, --output ', 'output file name').default(resolve(__dirname, '../../opensearch-openapi.yaml'))) .allowExcessArguments(false) .parse() diff --git a/tools/tests/linter/SchemasValidator.test.ts b/tools/tests/linter/SchemasValidator.test.ts new file mode 100644 index 000000000..cbd16a998 --- /dev/null +++ b/tools/tests/linter/SchemasValidator.test.ts @@ -0,0 +1,43 @@ +import SchemasValidator from '../../src/linter/SchemasValidator' + +test('validate() - named_schemas', () => { + const validator = new SchemasValidator('./tools/tests/linter/fixtures/schemas_validator/named_schemas') + expect(validator.validate()).toEqual([ + { + file: 'schemas/actions.yaml', + location: '#/components/schemas/Bark', + message: 'schema is invalid: data/type must be equal to one of the allowed values, data/type must be array, data/type must match a schema in anyOf' + }, + { + file: 'schemas/animals.yaml', + location: '#/components/schemas/Dog', + message: 'schema is invalid: data/type must be equal to one of the allowed values, data/type must be array, data/type must match a schema in anyOf' + } + ]) +}) + +test('validate() - anonymous_schemas', () => { + const validator = new SchemasValidator('./tools/tests/linter/fixtures/schemas_validator/anonymous_schemas') + expect(validator.validate()).toEqual([ + { + file: '_global_parameters.yaml', + location: 'human', + message: 'schema is invalid: data/type must be equal to one of the allowed values, data/type must be array, data/type must match a schema in anyOf' + }, + { + file: 'namespaces/_core.yaml', + location: '#/components/parameters/adopt::path.docket', + message: 'schema is invalid: data/type must be equal to one of the allowed values, data/type must be array, data/type must match a schema in anyOf' + }, + { + file: 'namespaces/adopt.yaml', + location: '#/components/requestBodies/adopt/content/application/json', + message: 'schema is invalid: data/type must be equal to one of the allowed values, data/type must be array, data/type must match a schema in anyOf' + }, + { + file: 'namespaces/_core.yaml', + location: '#/components/responses/adopt@200/content/application/json', + message: 'schema is invalid: data/type must be equal to one of the allowed values, data/type must be array, data/type must match a schema in anyOf' + } + ]) +}) diff --git a/tools/tests/linter/SpecValidator.test.ts b/tools/tests/linter/SpecValidator.test.ts index 0ba8cb576..7658ee860 100644 --- a/tools/tests/linter/SpecValidator.test.ts +++ b/tools/tests/linter/SpecValidator.test.ts @@ -2,11 +2,11 @@ import SpecValidator from 'linter/SpecValidator' test('validate()', () => { const validator = new SpecValidator('./tools/tests/linter/fixtures/empty') - expect(validator.validate()).toEqual([]) - validator.namespaces_folder.validate = jest.fn().mockReturnValue([{ file: 'namespaces/', message: 'namespace error' }]) validator.schemas_folder.validate = jest.fn().mockReturnValue([{ file: 'schemas/', message: 'schema error' }]) validator.schema_refs_validator.validate = jest.fn().mockReturnValue([{ file: 'schema_refs', message: 'schema refs error' }]) + validator.schemas_validator.validate = jest.fn().mockReturnValue([{ file: 'schemas/', message: 'schema error' }]) + validator.inline_object_schema_validator.validate = jest.fn().mockReturnValue([{ file: 'inline_file', message: 'inline_object_schema_validator error' }]) expect(validator.validate()).toEqual([ { file: 'namespaces/', message: 'namespace error' }, @@ -17,6 +17,8 @@ test('validate()', () => { validator.schemas_folder.validate = jest.fn().mockReturnValue([]) expect(validator.validate()).toEqual([ - { file: 'schema_refs', message: 'schema refs error' } + { file: 'schema_refs', message: 'schema refs error' }, + { file: 'inline_file', message: 'inline_object_schema_validator error' }, + { file: 'schemas/', message: 'schema error' } ]) }) diff --git a/tools/tests/linter/fixtures/schemas_validator/anonymous_schemas/_global_parameters.yaml b/tools/tests/linter/fixtures/schemas_validator/anonymous_schemas/_global_parameters.yaml new file mode 100644 index 000000000..479cfaaa4 --- /dev/null +++ b/tools/tests/linter/fixtures/schemas_validator/anonymous_schemas/_global_parameters.yaml @@ -0,0 +1,13 @@ +openapi: 3.1.0 +info: + title: '' + version: '' +components: + parameters: + human: + name: human + in: query + description: Whether to return human readable values for statistics. + schema: + type: bogus + default: true \ No newline at end of file diff --git a/tools/tests/linter/fixtures/schemas_validator/anonymous_schemas/_info.yaml b/tools/tests/linter/fixtures/schemas_validator/anonymous_schemas/_info.yaml new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/tools/tests/linter/fixtures/schemas_validator/anonymous_schemas/_info.yaml @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/tools/tests/linter/fixtures/schemas_validator/anonymous_schemas/_superseded_operations.yaml b/tools/tests/linter/fixtures/schemas_validator/anonymous_schemas/_superseded_operations.yaml new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/tools/tests/linter/fixtures/schemas_validator/anonymous_schemas/_superseded_operations.yaml @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/tools/tests/linter/fixtures/schemas_validator/anonymous_schemas/namespaces/shelter.yaml b/tools/tests/linter/fixtures/schemas_validator/anonymous_schemas/namespaces/shelter.yaml new file mode 100644 index 000000000..9eb26cb84 --- /dev/null +++ b/tools/tests/linter/fixtures/schemas_validator/anonymous_schemas/namespaces/shelter.yaml @@ -0,0 +1,54 @@ +openapi: 3.1.0 +info: + title: OpenSearch API + description: OpenSearch API + version: 1.0.0 +paths: + '/adopt/{animal}/dockets/{docket}': + get: + operationId: adopt.0 + parameters: + - $ref: '#/components/parameters/adopt::path.animal' + - $ref: '#/components/parameters/adopt::path.docket' + responses: + '200': + $ref: '#/components/responses/adopt@200' + post: + operationId: adopt.1 + parameters: + - $ref: '#/components/parameters/adopt::path.animal' + - $ref: '#/components/parameters/adopt::path.docket' + requestBody: + $ref: '#/components/requestBodies/adopt' + responses: + '200': + $ref: '#/components/responses/adopt@200' +components: + requestBodies: + adopt: { + content: { + application/json: { + schema: { + type: object2 + } + } + } + } + parameters: + adopt::path.animal: + name: animal + in: path + schema: + $ref: '../schemas/animals.yaml#/components/schemas/Animal' + adopt::path.docket: + name: docket + in: path + schema: + type: number2 + responses: + adopt@200: + description: '' + content: + application/json: + schema: + type: object2 \ No newline at end of file diff --git a/tools/tests/linter/fixtures/schemas_validator/anonymous_schemas/schemas/actions.yaml b/tools/tests/linter/fixtures/schemas_validator/anonymous_schemas/schemas/actions.yaml new file mode 100644 index 000000000..89bec0877 --- /dev/null +++ b/tools/tests/linter/fixtures/schemas_validator/anonymous_schemas/schemas/actions.yaml @@ -0,0 +1,11 @@ +openapi: 3.1.0 +info: + title: OpenSearch API + description: OpenSearch API + version: 1.0.0 +components: + schemas: + Bark: + type: string + Meow: + type: string \ No newline at end of file diff --git a/tools/tests/linter/fixtures/schemas_validator/anonymous_schemas/schemas/animals.yaml b/tools/tests/linter/fixtures/schemas_validator/anonymous_schemas/schemas/animals.yaml new file mode 100644 index 000000000..bdb82a16f --- /dev/null +++ b/tools/tests/linter/fixtures/schemas_validator/anonymous_schemas/schemas/animals.yaml @@ -0,0 +1,21 @@ +openapi: 3.1.0 +info: + title: OpenSearch API + description: OpenSearch API + version: 1.0.0 +components: + schemas: + Animal: + oneOf: + - $ref: '#/components/schemas/Dog' + - $ref: '#/components/schemas/Cat' + Dog: + type: object + properties: + bark: + $ref: 'actions.yaml#/components/schemas/Bark' + Cat: + type: object + properties: + meow: + $ref: 'actions.yaml#/components/schemas/Meow' \ No newline at end of file diff --git a/tools/tests/linter/fixtures/schemas_validator/named_schemas/_global_parameters.yaml b/tools/tests/linter/fixtures/schemas_validator/named_schemas/_global_parameters.yaml new file mode 100644 index 000000000..479cfaaa4 --- /dev/null +++ b/tools/tests/linter/fixtures/schemas_validator/named_schemas/_global_parameters.yaml @@ -0,0 +1,13 @@ +openapi: 3.1.0 +info: + title: '' + version: '' +components: + parameters: + human: + name: human + in: query + description: Whether to return human readable values for statistics. + schema: + type: bogus + default: true \ No newline at end of file diff --git a/tools/tests/linter/fixtures/schemas_validator/named_schemas/_info.yaml b/tools/tests/linter/fixtures/schemas_validator/named_schemas/_info.yaml new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/tools/tests/linter/fixtures/schemas_validator/named_schemas/_info.yaml @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/tools/tests/linter/fixtures/schemas_validator/named_schemas/_superseded_operations.yaml b/tools/tests/linter/fixtures/schemas_validator/named_schemas/_superseded_operations.yaml new file mode 100644 index 000000000..9e26dfeeb --- /dev/null +++ b/tools/tests/linter/fixtures/schemas_validator/named_schemas/_superseded_operations.yaml @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/tools/tests/linter/fixtures/schemas_validator/named_schemas/namespaces/shelter.yaml b/tools/tests/linter/fixtures/schemas_validator/named_schemas/namespaces/shelter.yaml new file mode 100644 index 000000000..0f1a2970b --- /dev/null +++ b/tools/tests/linter/fixtures/schemas_validator/named_schemas/namespaces/shelter.yaml @@ -0,0 +1,45 @@ +openapi: 3.1.0 +info: + title: OpenSearch API + description: OpenSearch API + version: 1.0.0 +paths: + '/adopt/{animal}/dockets/{docket}': + get: + operationId: adopt.0 + parameters: + - $ref: '#/components/parameters/adopt::path.animal' + - $ref: '#/components/parameters/adopt::path.docket' + responses: + '200': + $ref: '#/components/responses/adopt@200' + post: + operationId: adopt.1 + parameters: + - $ref: '#/components/parameters/adopt::path.animal' + - $ref: '#/components/parameters/adopt::path.docket' + requestBody: + $ref: '#/components/requestBodies/adopt' + responses: + '200': + $ref: '#/components/responses/adopt@200' +components: + requestBodies: + adopt: {} + parameters: + adopt::path.animal: + name: animal + in: path + schema: + $ref: '../schemas/animals.yaml#/components/schemas/Animal' + adopt::path.docket: + name: docket + in: path + schema: + type: number + responses: + adopt@200: + description: '' + application/json: + schema: + type: object \ No newline at end of file diff --git a/tools/tests/linter/fixtures/schemas_validator/named_schemas/schemas/actions.yaml b/tools/tests/linter/fixtures/schemas_validator/named_schemas/schemas/actions.yaml new file mode 100644 index 000000000..24dbbb8d6 --- /dev/null +++ b/tools/tests/linter/fixtures/schemas_validator/named_schemas/schemas/actions.yaml @@ -0,0 +1,11 @@ +openapi: 3.1.0 +info: + title: OpenSearch API + description: OpenSearch API + version: 1.0.0 +components: + schemas: + Bark: + type: bogus + Meow: + type: string \ No newline at end of file diff --git a/tools/tests/linter/fixtures/schemas_validator/named_schemas/schemas/animals.yaml b/tools/tests/linter/fixtures/schemas_validator/named_schemas/schemas/animals.yaml new file mode 100644 index 000000000..6bf0f10f5 --- /dev/null +++ b/tools/tests/linter/fixtures/schemas_validator/named_schemas/schemas/animals.yaml @@ -0,0 +1,21 @@ +openapi: 3.1.0 +info: + title: OpenSearch API + description: OpenSearch API + version: 1.0.0 +components: + schemas: + Animal: + oneOf: + - $ref: '#/components/schemas/Dog' + - $ref: '#/components/schemas/Cat' + Dog: + type: bogus + properties: + bark: + $ref: 'actions.yaml#/components/schemas/Bark' + Cat: + type: object + properties: + meow: + $ref: 'actions.yaml#/components/schemas/Meow' \ No newline at end of file