[Security Solution] Write additional metadata when generating types u…

…sing `kbn-openapi-generator` (#174718) ## Summary Follow up PR from #174317 with the following fixes/enhancements to `kbn-openapi-generator`: * Fix extraneous `.`'s in paths causing generated files to not be written to disk * Updates `README.md` for latest method of adding CI actions * Adds `info` details to generated metadata comment for more easily tracing back to source schema * Moves assistant `*.schema.yaml` files from `elastic_assistant` plugin to `kbn-elastic-assistant-common` package > [!NOTE] > This PR includes a manual run of the `kbn-elastic-assistant-common` package `yarn openapi:generate` script as a reference example. Since this PR also updates the generation template to include the `info` metadata, CI will run the generator for the other consumers (`security_solution` & `osquery`) automatically, and commit those updates to this PR. <img width="16" src="https://user-images.githubusercontent.com/2946766/160040365-b1b8bb8a-d2d7-4187-b9b9-04817f8e2ae5.gif" /> ### Test instructions You can test against the `kbn-elastic-assistant-common` package using either the main CLI script from kibana root: ``` node scripts/generate_openapi --rootDir ./x-pack/packages/kbn-elastic-assistant-common ``` or via the yarn command: ``` cd x-pack/packages/kbn-elastic-assistant-common/ yarn openapi-generate ``` ### Checklist Delete any items that are not applicable to this PR. - [X] [Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html) was added for features that require explanation or tutorials --------- Co-authored-by: kibanamachine <[email protected]>
elastic · Jan 18, 2024 · 98960f2 · 98960f2
1 parent 35029bc
commit 98960f2
Show file tree

Hide file tree

Showing 94 changed files with 384 additions and 24 deletions.
diff --git a/.buildkite/scripts/steps/code_generation/elastic_assistant_codegen.sh b/.buildkite/scripts/steps/code_generation/elastic_assistant_codegen.sh
@@ -6,5 +6,5 @@ source .buildkite/scripts/common/util.sh
 
 echo --- Elastic Assistant OpenAPI Code Generation
 
-(cd x-pack/plugins/elastic_assistant && yarn openapi:generate)
+(cd x-pack/packages/kbn-elastic-assistant-common && yarn openapi:generate)
 check_for_changed_files "yarn openapi:generate" true
diff --git a/packages/kbn-openapi-generator/README.md b/packages/kbn-openapi-generator/README.md
@@ -72,6 +72,10 @@ import { z } from 'zod';
 /*
  * NOTICE: Do not edit this file manually.
  * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
+ * 
+ * info:
+ *   title: Install Prebuilt Rules API endpoint
+ *   version: 1
  */
 
 export type InstallPrebuiltRulesResponse = z.infer<typeof InstallPrebuiltRulesResponse>;
@@ -134,17 +138,18 @@ echo --- Security Solution OpenAPI Code Generation
 check_for_changed_files "yarn openapi:generate" true
 ```
 
-This scripts sets up the minimal environment required fro code generation and runs the code generation script. Then it checks if there are any changes and commits them if there are any using the `check_for_changed_files` function.
+This script sets up the minimal environment required for code generation and runs the code generation script. Then it checks if there are any changes and commits them if there are any using the `check_for_changed_files` function.
 
-Then add the code generation script to your plugin build pipeline. Open your plugin build pipeline, for example `.buildkite/pipelines/pull_request/base.yml`, and add the following command to the steps list adjusting the path to your code generation script:
+Then add the code generation script to the build pipeline. Open the buildkite checks at`.buildkite/scripts/steps/checks.sh`, and add the path to your code generation script:
 
-```yaml
-- command: .buildkite/scripts/steps/code_generation/security_solution_codegen.sh
-  label: 'Security Solution OpenAPI codegen'
-  agents:
-    queue: n2-2-spot
-  timeout_in_minutes: 60
-  parallelism: 1
+```sh
+...
+.buildkite/scripts/steps/checks/saved_objects_definition_change.sh
+.buildkite/scripts/steps/code_generation/elastic_assistant_codegen.sh
+.buildkite/scripts/steps/code_generation/security_solution_codegen.sh
+.buildkite/scripts/steps/code_generation/osquery_codegen.sh
+.buildkite/scripts/steps/checks/yarn_deduplicate.sh
+...
 ```
 
 Now on every pull request the code generation script will run and commit the changes if there are any.

diff --git a/packages/kbn-openapi-generator/src/lib/get_generated_file_path.ts b/packages/kbn-openapi-generator/src/lib/get_generated_file_path.ts
@@ -7,5 +7,7 @@
  */
 
 export function getGeneratedFilePath(sourcePath: string) {
-  return sourcePath.replace(/\..+$/, '.gen.ts');
+  // Remove any double extension like `.schema.yaml` or `.schema.yml` and replace with `.gen.ts`
+  const secondToLastDot = sourcePath.lastIndexOf('.', sourcePath.lastIndexOf('.') - 1);
+  return `${sourcePath.substring(0, secondToLastDot)}.gen.ts`;
 }
diff --git a/packages/kbn-openapi-generator/src/parser/get_generation_context.ts b/packages/kbn-openapi-generator/src/parser/get_generation_context.ts
@@ -12,10 +12,12 @@ import { getComponents } from './lib/get_components';
 import { getImportsMap, ImportsMap } from './lib/get_imports_map';
 import { normalizeSchema } from './lib/normalize_schema';
 import { NormalizedOperation, OpenApiDocument } from './openapi_types';
+import { getInfo } from './lib/get_info';
 
 export interface GenerationContext {
   components: OpenAPIV3.ComponentsObject | undefined;
   operations: NormalizedOperation[];
+  info: OpenAPIV3.InfoObject;
   imports: ImportsMap;
 }
 
@@ -24,11 +26,13 @@ export function getGenerationContext(document: OpenApiDocument): GenerationConte
 
   const components = getComponents(normalizedDocument);
   const operations = getApiOperationsList(normalizedDocument);
+  const info = getInfo(normalizedDocument);
   const imports = getImportsMap(normalizedDocument);
 
   return {
     components,
     operations,
+    info,
     imports,
   };
 }
diff --git a/packages/kbn-openapi-generator/src/parser/lib/get_info.ts b/packages/kbn-openapi-generator/src/parser/lib/get_info.ts
@@ -0,0 +1,13 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import type { OpenApiDocument } from '../openapi_types';
+
+export function getInfo(parsedSchema: OpenApiDocument) {
+  return parsedSchema.info;
+}
diff --git a/packages/kbn-openapi-generator/src/template_service/templates/disclaimer.handlebars b/packages/kbn-openapi-generator/src/template_service/templates/disclaimer.handlebars
@@ -1,5 +1,8 @@
 /*
  * NOTICE: Do not edit this file manually.
  * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
+ *
+ * info:
+ *   title: {{info.title}}
+ *   version: {{info.version}}
  */
-
diff --git a/x-pack/packages/kbn-elastic-assistant-common/README.md b/x-pack/packages/kbn-elastic-assistant-common/README.md
@@ -18,3 +18,19 @@ To (interactively) run unit tests with code coverage, run the following command:
 ```sh
 cd $KIBANA_HOME && node scripts/jest --watch x-pack/packages/kbn-elastic-assistant-common --coverage
 ```
+
+## OpenAPI Codegen
+
+Implemented using the new OpenAPI codegen and bundle packages:
+* Includes OpenAPI codegen script and CI action as detailed in: https://github.com/elastic/kibana/pull/166269
+* Includes OpenAPI docs bundling script as detailed in: https://github.com/elastic/kibana/pull/171526
+
+To run codegen/bundling locally, cd to `x-pack/packages/kbn-elastic-assistant-common/` and run any of the following commands:
+
+```bash
+yarn openapi:generate
+yarn openapi:generate:debug
+yarn openapi:bundle
+```
+
+Codegen is configured to run on CI by means of the `.buildkite/scripts/steps/code_generation/elastic_assistant_codegen.sh` script, which is run as part of the `checks` pipeline, and is registered in `.buildkite/scripts/steps/checks.sh`.
diff --git a/...apabilities/get_capabilities_route.gen.ts → ...apabilities/get_capabilities_route.gen.ts b/...apabilities/get_capabilities_route.gen.ts → ...apabilities/get_capabilities_route.gen.ts
@@ -10,6 +10,10 @@ import { z } from 'zod';
 /*
  * NOTICE: Do not edit this file manually.
  * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
+ *
+ * info:
+ *   title: Get Capabilities API endpoint
+ *   version: 1
  */
 
 export type GetCapabilitiesResponse = z.infer<typeof GetCapabilitiesResponse>;

diff --git a/...lities/get_capabilities_route.schema.yaml → ...lities/get_capabilities_route.schema.yaml b/...lities/get_capabilities_route.schema.yaml → ...lities/get_capabilities_route.schema.yaml
diff --git a/x-pack/packages/kbn-elastic-assistant-common/index.ts b/x-pack/packages/kbn-elastic-assistant-common/index.ts
@@ -5,6 +5,8 @@
  * 2.0.
  */
 
+export { GetCapabilitiesResponse } from './impl/schemas/capabilities/get_capabilities_route.gen';
+
 export { defaultAssistantFeatures } from './impl/capabilities';
 export type { AssistantFeatures } from './impl/capabilities';
 

diff --git a/x-pack/packages/kbn-elastic-assistant-common/package.json b/x-pack/packages/kbn-elastic-assistant-common/package.json
@@ -3,5 +3,10 @@
   "private": true,
   "version": "1.0.0",
   "license": "Elastic License 2.0",
-  "sideEffects": false
+  "sideEffects": false,
+  "scripts": {
+    "openapi:generate": "node scripts/openapi/generate",
+    "openapi:generate:debug": "node --inspect-brk scripts/openapi/generate",
+    "openapi:bundle": "node scripts/openapi/bundle"
+  }
 }
diff --git a/...astic_assistant/scripts/openapi/bundle.js → ...ssistant-common/scripts/openapi/bundle.js b/...astic_assistant/scripts/openapi/bundle.js → ...ssistant-common/scripts/openapi/bundle.js
@@ -7,12 +7,13 @@
 
 require('../../../../../src/setup_node_env');
 const { bundle } = require('@kbn/openapi-bundler');
+// eslint-disable-next-line import/no-nodejs-modules
 const { resolve } = require('path');
 
 const ELASTIC_ASSISTANT_ROOT = resolve(__dirname, '../..');
 
 bundle({
   rootDir: ELASTIC_ASSISTANT_ROOT,
-  sourceGlob: './server/schemas/**/*.schema.yaml',
+  sourceGlob: './impl/schemas/**/*.schema.yaml',
   outputFilePath: './target/openapi/elastic_assistant.bundled.schema.yaml',
 });
diff --git a/...tic_assistant/scripts/openapi/generate.js → ...istant-common/scripts/openapi/generate.js b/...tic_assistant/scripts/openapi/generate.js → ...istant-common/scripts/openapi/generate.js
@@ -7,12 +7,13 @@
 
 require('../../../../../src/setup_node_env');
 const { generate } = require('@kbn/openapi-generator');
+// eslint-disable-next-line import/no-nodejs-modules
 const { resolve } = require('path');
 
 const ELASTIC_ASSISTANT_ROOT = resolve(__dirname, '../..');
 
 generate({
   rootDir: ELASTIC_ASSISTANT_ROOT,
-  sourceGlob: './server/schemas/**/*.schema.yaml',
+  sourceGlob: './impl/schemas/**/*.schema.yaml',
   templateName: 'zod_operation_schema',
 });
diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/capabilities/capabilities.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/capabilities/capabilities.tsx
@@ -6,15 +6,13 @@
  */
 
 import { HttpSetup, IHttpFetchError } from '@kbn/core-http-browser';
-import { AssistantFeatures } from '@kbn/elastic-assistant-common';
+import { GetCapabilitiesResponse } from '@kbn/elastic-assistant-common';
 
 export interface GetCapabilitiesParams {
   http: HttpSetup;
   signal?: AbortSignal | undefined;
 }
 
-export type GetCapabilitiesResponse = AssistantFeatures;
-
 /**
  * API call for fetching assistant capabilities
  *

diff --git a/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/capabilities/use_capabilities.tsx b/x-pack/packages/kbn-elastic-assistant/impl/assistant/api/capabilities/use_capabilities.tsx
@@ -10,7 +10,8 @@ import { useQuery } from '@tanstack/react-query';
 import type { HttpSetup, IHttpFetchError, ResponseErrorBody } from '@kbn/core-http-browser';
 import type { IToasts } from '@kbn/core-notifications-browser';
 import { i18n } from '@kbn/i18n';
-import { getCapabilities, GetCapabilitiesResponse } from './capabilities';
+import type { GetCapabilitiesResponse } from '@kbn/elastic-assistant-common';
+import { getCapabilities } from './capabilities';
 
 const CAPABILITIES_QUERY_KEY = ['elastic-assistant', 'capabilities'];
 

diff --git a/x-pack/plugins/elastic_assistant/package.json b/x-pack/plugins/elastic_assistant/package.json
@@ -5,9 +5,6 @@
   "private": true,
   "license": "Elastic License 2.0",
   "scripts": {
-    "evaluate-model": "node ./scripts/model_evaluator",
-    "openapi:generate": "node scripts/openapi/generate",
-    "openapi:generate:debug": "node --inspect-brk scripts/openapi/generate",
-    "openapi:bundle": "node scripts/openapi/bundle"
+    "evaluate-model": "node ./scripts/model_evaluator"
   }
 }
diff --git a/x-pack/plugins/elastic_assistant/server/routes/capabilities/get_capabilities_route.ts b/x-pack/plugins/elastic_assistant/server/routes/capabilities/get_capabilities_route.ts
@@ -8,10 +8,10 @@
 import { IKibanaResponse, IRouter } from '@kbn/core/server';
 import { transformError } from '@kbn/securitysolution-es-utils';
 
+import type { GetCapabilitiesResponse } from '@kbn/elastic-assistant-common';
 import { CAPABILITIES } from '../../../common/constants';
 import { ElasticAssistantRequestHandlerContext } from '../../types';
 
-import { GetCapabilitiesResponse } from '../../schemas/capabilities/get_capabilities_route.gen';
 import { buildResponse } from '../../lib/build_response';
 import { DEFAULT_PLUGIN_NAME, getPluginNameFromRequest } from '../helpers';
 

diff --git a/x-pack/plugins/osquery/common/api/asset/assets_status.gen.ts b/x-pack/plugins/osquery/common/api/asset/assets_status.gen.ts
@@ -10,6 +10,10 @@ import { z } from 'zod';
 /*
  * NOTICE: Do not edit this file manually.
  * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
+ *
+ * info:
+ *   title: Assets Status Schema
+ *   version: 1
  */
 
 export type AssetsRequestQuery = z.infer<typeof AssetsRequestQuery>;

diff --git a/x-pack/plugins/osquery/common/api/fleet_wrapper/get_agent_details.gen.ts b/x-pack/plugins/osquery/common/api/fleet_wrapper/get_agent_details.gen.ts
@@ -10,6 +10,10 @@ import { z } from 'zod';
 /*
  * NOTICE: Do not edit this file manually.
  * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
+ *
+ * info:
+ *   title: Get agent details schema
+ *   version: 1
  */
 
 export type GetAgentDetailsRequestParams = z.infer<typeof GetAgentDetailsRequestParams>;

diff --git a/x-pack/plugins/osquery/common/api/fleet_wrapper/get_agent_policies.gen.ts b/x-pack/plugins/osquery/common/api/fleet_wrapper/get_agent_policies.gen.ts
@@ -10,6 +10,10 @@ import { z } from 'zod';
 /*
  * NOTICE: Do not edit this file manually.
  * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
+ *
+ * info:
+ *   title: Get agent policies schema
+ *   version: 1
  */
 
 export type GetAgentPoliciesRequestParams = z.infer<typeof GetAgentPoliciesRequestParams>;

diff --git a/x-pack/plugins/osquery/common/api/fleet_wrapper/get_agent_policy.gen.ts b/x-pack/plugins/osquery/common/api/fleet_wrapper/get_agent_policy.gen.ts
@@ -10,6 +10,10 @@ import { z } from 'zod';
 /*
  * NOTICE: Do not edit this file manually.
  * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
+ *
+ * info:
+ *   title: Get agent policy schema
+ *   version: 1
  */
 
 import { Id } from '../model/schema/common_attributes.gen';

diff --git a/x-pack/plugins/osquery/common/api/fleet_wrapper/get_agent_status.gen.ts b/x-pack/plugins/osquery/common/api/fleet_wrapper/get_agent_status.gen.ts
@@ -10,6 +10,10 @@ import { z } from 'zod';
 /*
  * NOTICE: Do not edit this file manually.
  * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
+ *
+ * info:
+ *   title: Get agent status schema
+ *   version: 1
  */
 
 import { KueryOrUndefined, Id } from '../model/schema/common_attributes.gen';

diff --git a/x-pack/plugins/osquery/common/api/fleet_wrapper/get_agents.gen.ts b/x-pack/plugins/osquery/common/api/fleet_wrapper/get_agents.gen.ts
@@ -10,6 +10,10 @@ import { z } from 'zod';
 /*
  * NOTICE: Do not edit this file manually.
  * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
+ *
+ * info:
+ *   title: Get agents schema
+ *   version: 1
  */
 
 export type GetAgentsRequestParams = z.infer<typeof GetAgentsRequestParams>;

diff --git a/x-pack/plugins/osquery/common/api/fleet_wrapper/get_package_policies.gen.ts b/x-pack/plugins/osquery/common/api/fleet_wrapper/get_package_policies.gen.ts
@@ -10,6 +10,10 @@ import { z } from 'zod';
 /*
  * NOTICE: Do not edit this file manually.
  * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
+ *
+ * info:
+ *   title: Get package policies schema
+ *   version: 1
  */
 
 export type GetPackagePoliciesRequestParams = z.infer<typeof GetPackagePoliciesRequestParams>;

diff --git a/x-pack/plugins/osquery/common/api/live_query/create_live_query.gen.ts b/x-pack/plugins/osquery/common/api/live_query/create_live_query.gen.ts
@@ -10,6 +10,10 @@ import { z } from 'zod';
 /*
  * NOTICE: Do not edit this file manually.
  * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
+ *
+ * info:
+ *   title: Create Live Query Schema
+ *   version: 2023-10-31
  */
 
 import {

diff --git a/x-pack/plugins/osquery/common/api/live_query/find_live_query.gen.ts b/x-pack/plugins/osquery/common/api/live_query/find_live_query.gen.ts
@@ -10,6 +10,10 @@ import { z } from 'zod';
 /*
  * NOTICE: Do not edit this file manually.
  * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
+ *
+ * info:
+ *   title: Find Live Queries Schema
+ *   version: 2023-10-31
  */
 
 import {

diff --git a/x-pack/plugins/osquery/common/api/live_query/get_live_query_details.gen.ts b/x-pack/plugins/osquery/common/api/live_query/get_live_query_details.gen.ts
@@ -10,6 +10,10 @@ import { z } from 'zod';
 /*
  * NOTICE: Do not edit this file manually.
  * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
+ *
+ * info:
+ *   title: Get Live Query Details Schema
+ *   version: 2023-10-31
  */
 
 export type SuccessResponse = z.infer<typeof SuccessResponse>;

diff --git a/x-pack/plugins/osquery/common/api/live_query/get_live_query_results.gen.ts b/x-pack/plugins/osquery/common/api/live_query/get_live_query_results.gen.ts
@@ -10,6 +10,10 @@ import { z } from 'zod';
 /*
  * NOTICE: Do not edit this file manually.
  * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
+ *
+ * info:
+ *   title: Get Live Query Results Schema
+ *   version: 2023-10-31
  */
 
 import {

diff --git a/x-pack/plugins/osquery/common/api/model/schema/common_attributes.gen.ts b/x-pack/plugins/osquery/common/api/model/schema/common_attributes.gen.ts
@@ -10,6 +10,10 @@ import { z } from 'zod';
 /*
  * NOTICE: Do not edit this file manually.
  * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
+ *
+ * info:
+ *   title: Common Osquery Attributes
+ *   version: 2023-10-31
  */
 
 export type Id = z.infer<typeof Id>;

diff --git a/x-pack/plugins/osquery/common/api/packs/create_pack.gen.ts b/x-pack/plugins/osquery/common/api/packs/create_pack.gen.ts
@@ -10,6 +10,10 @@ import { z } from 'zod';
 /*
  * NOTICE: Do not edit this file manually.
  * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
+ *
+ * info:
+ *   title: Create Pack Schema
+ *   version: 2023-10-31
  */
 
 import {

diff --git a/x-pack/plugins/osquery/common/api/packs/delete_packs.gen.ts b/x-pack/plugins/osquery/common/api/packs/delete_packs.gen.ts
@@ -10,6 +10,10 @@ import { z } from 'zod';
 /*
  * NOTICE: Do not edit this file manually.
  * This file is automatically generated by the OpenAPI Generator, @kbn/openapi-generator.
+ *
+ * info:
+ *   title: Delete Saved Queries Schema
+ *   version: 2023-10-31
  */
 
 import { PackId } from '../model/schema/common_attributes.gen';
-Original file line number
+Diff line change
@@ Expand Up / @@ -5,6 +5,8 @@ @@
      * 2.0.
      */
+    export { GetCapabilitiesResponse } from './impl/schemas/capabilities/get_capabilities_route.gen';
     export { defaultAssistantFeatures } from './impl/capabilities';
     export type { AssistantFeatures } from './impl/capabilities';
@@ Expand Down @@