Skip to content

Commit

Permalink
[8.x] [IndexAdapter] Extract index-adapter package from data-stream-a…
Browse files Browse the repository at this point in the history 
…dapter (#199575) (#199848)

# Backport

This will backport the following commits from `main` to `8.x`:
- [[IndexAdapter] Extract index-adapter package from data-stream-adapter
(#199575)](#199575)

<!--- Backport version: 8.9.8 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Sergi
Massaneda","email":"[email protected]"},"sourceCommit":{"committedDate":"2024-11-12T17:16:32Z","message":"[IndexAdapter]
Extract index-adapter package from data-stream-adapter (#199575)\n\n##
Summary\r\n\r\nExtracts `IndexAdapter` from `DataStreamAdapter`
and\r\n`IndexPatternAdapter` from `DataStreamSpaceAdapter`.\r\n\r\nThere
are no breaking changes for the _data-stream-adapter_ package;
the\r\nbehavior of both the `DataStreamAdapter` and
`DataStreamSpaceAdapter`\r\nremains unchanged.\r\n\r\nThe new
_index-adapter_ package exports `IndexAdapter`
and\r\n`IndexPatternAdapter` to manage individual indices without using
data\r\nstreams.\r\n\r\nThis is needed for SIEM rule
migrations.\r\n\r\n---------\r\n\r\nCo-authored-by: kibanamachine
<[email protected]>\r\nCo-authored-by:
Elastic Machine
<[email protected]>","sha":"9a9f02c9315beda4089b1ef16089747c080bc345","branchLabelMapping":{"^v9.0.0$":"main","^v8.17.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","v9.0.0","Team:Threat
Hunting","Team:
SecuritySolution","backport:prev-minor","v8.18.0"],"number":199575,"url":"https://github.com/elastic/kibana/pull/199575","mergeCommit":{"message":"[IndexAdapter]
Extract index-adapter package from data-stream-adapter (#199575)\n\n##
Summary\r\n\r\nExtracts `IndexAdapter` from `DataStreamAdapter`
and\r\n`IndexPatternAdapter` from `DataStreamSpaceAdapter`.\r\n\r\nThere
are no breaking changes for the _data-stream-adapter_ package;
the\r\nbehavior of both the `DataStreamAdapter` and
`DataStreamSpaceAdapter`\r\nremains unchanged.\r\n\r\nThe new
_index-adapter_ package exports `IndexAdapter`
and\r\n`IndexPatternAdapter` to manage individual indices without using
data\r\nstreams.\r\n\r\nThis is needed for SIEM rule
migrations.\r\n\r\n---------\r\n\r\nCo-authored-by: kibanamachine
<[email protected]>\r\nCo-authored-by:
Elastic Machine
<[email protected]>","sha":"9a9f02c9315beda4089b1ef16089747c080bc345"}},"sourceBranch":"main","suggestedTargetBranches":["8.18"],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","labelRegex":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/199575","number":199575,"mergeCommit":{"message":"[IndexAdapter]
Extract index-adapter package from data-stream-adapter (#199575)\n\n##
Summary\r\n\r\nExtracts `IndexAdapter` from `DataStreamAdapter`
and\r\n`IndexPatternAdapter` from `DataStreamSpaceAdapter`.\r\n\r\nThere
are no breaking changes for the _data-stream-adapter_ package;
the\r\nbehavior of both the `DataStreamAdapter` and
`DataStreamSpaceAdapter`\r\nremains unchanged.\r\n\r\nThe new
_index-adapter_ package exports `IndexAdapter`
and\r\n`IndexPatternAdapter` to manage individual indices without using
data\r\nstreams.\r\n\r\nThis is needed for SIEM rule
migrations.\r\n\r\n---------\r\n\r\nCo-authored-by: kibanamachine
<[email protected]>\r\nCo-authored-by:
Elastic Machine
<[email protected]>","sha":"9a9f02c9315beda4089b1ef16089747c080bc345"}},{"branch":"8.18","label":"v8.18.0","labelRegex":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->
  • Loading branch information
@semd
semd authored Nov 12, 2024
1 parent 7ecc654 commit 953bc0f
Show file tree
Hide file tree
Showing 34 changed files with 858 additions and 221 deletions.
1 change: 1 addition & 0 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -567,6 +567,7 @@
"@kbn/i18n-react": "link:packages/kbn-i18n-react",
"@kbn/iframe-embedded-plugin": "link:x-pack/test/functional_embedded/plugins/iframe_embedded",
"@kbn/image-embeddable-plugin": "link:src/plugins/image_embeddable",
"@kbn/index-adapter": "link:packages/kbn-index-adapter",
"@kbn/index-lifecycle-management-plugin": "link:x-pack/plugins/index_lifecycle_management",
"@kbn/index-management-plugin": "link:x-pack/plugins/index_management",
"@kbn/index-management-shared-types": "link:x-pack/packages/index-management/index_management_shared_types",
Expand Down
10 changes: 5 additions & 5 deletions packages/kbn-data-stream-adapter/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,13 @@

export { DataStreamAdapter } from './src/data_stream_adapter';
export { DataStreamSpacesAdapter } from './src/data_stream_spaces_adapter';
export { retryTransientEsErrors } from './src/retry_transient_es_errors';
export { ecsFieldMap, type EcsFieldMap } from './src/field_maps/ecs_field_map';

export { retryTransientEsErrors, ecsFieldMap } from '@kbn/index-adapter';
export type {
DataStreamAdapterParams,
SetComponentTemplateParams,
SetIndexTemplateParams,
InstallParams,
} from './src/data_stream_adapter';
export * from './src/field_maps/types';
EcsFieldMap,
} from '@kbn/index-adapter';

export * from '@kbn/index-adapter/src/field_maps/types';
5 changes: 3 additions & 2 deletions packages/kbn-data-stream-adapter/kibana.jsonc
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
{
"type": "shared-common",
"type": "shared-server",
"id": "@kbn/data-stream-adapter",
"owner": "@elastic/security-threat-hunting-explore"
"owner": "@elastic/security-threat-hunting",
"visibility": "shared"
}
3 changes: 2 additions & 1 deletion packages/kbn-data-stream-adapter/src/create_or_update_data_stream.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,10 +136,11 @@ describe('createOrUpdateDataStream', () => {
it(`should create data stream if not exists`, async () => {
esClient.indices.getDataStream.mockResolvedValueOnce({ data_streams: [] });

await createDataStream({
await createOrUpdateDataStream({
esClient,
logger,
name,
totalFieldsLimit,
});

expect(esClient.indices.createDataStream).toHaveBeenCalledWith({ name });
Expand Down
5 changes: 3 additions & 2 deletions packages/kbn-data-stream-adapter/src/create_or_update_data_stream.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import type { IndicesDataStream } from '@elastic/elasticsearch/lib/api/types';
import type { IndicesSimulateIndexTemplateResponse } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import type { Logger, ElasticsearchClient } from '@kbn/core/server';
import { get } from 'lodash';
import { retryTransientEsErrors } from './retry_transient_es_errors';
import { retryTransientEsErrors } from '@kbn/index-adapter';

interface UpdateIndexMappingsOpts {
logger: Logger;
Expand Down Expand Up @@ -168,7 +168,7 @@ export async function createDataStream({
esClient,
name,
}: CreateDataStreamParams): Promise<void> {
logger.info(`Creating data stream - ${name}`);
logger.debug(`Checking data stream exists - ${name}`);

// check if data stream exists
let dataStreamExists = false;
Expand All @@ -189,6 +189,7 @@ export async function createDataStream({
if (dataStreamExists) {
return;
}
logger.info(`Installing data stream - ${name}`);

try {
await retryTransientEsErrors(() => esClient.indices.createDataStream({ name }), { logger });
Expand Down
139 changes: 8 additions & 131 deletions packages/kbn-data-stream-adapter/src/data_stream_adapter.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,145 +7,22 @@
* License v3.0 only", or the "Server Side Public License, v 1".
*/

import type {
ClusterPutComponentTemplateRequest,
IndicesIndexSettings,
IndicesPutIndexTemplateIndexTemplateMapping,
IndicesPutIndexTemplateRequest,
} from '@elastic/elasticsearch/lib/api/types';
import type { Logger, ElasticsearchClient } from '@kbn/core/server';
import type { Subject } from 'rxjs';
import type { FieldMap } from './field_maps/types';
import { createOrUpdateComponentTemplate } from './create_or_update_component_template';
import { IndexAdapter, SetIndexTemplateParams, type InstallParams } from '@kbn/index-adapter';
import { createOrUpdateDataStream } from './create_or_update_data_stream';
import { createOrUpdateIndexTemplate } from './create_or_update_index_template';
import { InstallShutdownError, installWithTimeout } from './install_with_timeout';
import { getComponentTemplate, getIndexTemplate } from './resource_installer_utils';

export interface DataStreamAdapterParams {
kibanaVersion: string;
totalFieldsLimit?: number;
}
export interface SetComponentTemplateParams {
name: string;
fieldMap: FieldMap;
settings?: IndicesIndexSettings;
dynamic?: 'strict' | boolean;
}
export interface SetIndexTemplateParams {
name: string;
componentTemplateRefs?: string[];
namespace?: string;
template?: IndicesPutIndexTemplateIndexTemplateMapping;
hidden?: boolean;
}

export interface GetInstallFnParams {
logger: Logger;
pluginStop$: Subject<void>;
tasksTimeoutMs?: number;
}
export interface InstallParams {
logger: Logger;
esClient: ElasticsearchClient | Promise<ElasticsearchClient>;
pluginStop$: Subject<void>;
tasksTimeoutMs?: number;
}

const DEFAULT_FIELDS_LIMIT = 2500;

export class DataStreamAdapter {
protected readonly kibanaVersion: string;
protected readonly totalFieldsLimit: number;
protected componentTemplates: ClusterPutComponentTemplateRequest[] = [];
protected indexTemplates: IndicesPutIndexTemplateRequest[] = [];
protected installed: boolean;

constructor(protected readonly name: string, options: DataStreamAdapterParams) {
this.installed = false;
this.kibanaVersion = options.kibanaVersion;
this.totalFieldsLimit = options.totalFieldsLimit ?? DEFAULT_FIELDS_LIMIT;
}

public setComponentTemplate(params: SetComponentTemplateParams) {
if (this.installed) {
throw new Error('Cannot set component template after install');
}
this.componentTemplates.push(getComponentTemplate(params));
}

export class DataStreamAdapter extends IndexAdapter {
public setIndexTemplate(params: SetIndexTemplateParams) {
if (this.installed) {
throw new Error('Cannot set index template after install');
}
this.indexTemplates.push(
getIndexTemplate({
...params,
indexPatterns: [this.name],
kibanaVersion: this.kibanaVersion,
totalFieldsLimit: this.totalFieldsLimit,
})
);
}

protected getInstallFn({ logger, pluginStop$, tasksTimeoutMs }: GetInstallFnParams) {
return async (promise: Promise<void>, description?: string): Promise<void> => {
try {
await installWithTimeout({
installFn: () => promise,
description,
timeoutMs: tasksTimeoutMs,
pluginStop$,
});
} catch (err) {
if (err instanceof InstallShutdownError) {
logger.info(err.message);
} else {
throw err;
}
}
};
super.setIndexTemplate({ ...params, isDataStream: true });
}

public async install({
logger,
esClient: esClientToResolve,
pluginStop$,
tasksTimeoutMs,
}: InstallParams) {
public async install(params: InstallParams) {
this.installed = true;
const { logger, pluginStop$, tasksTimeoutMs } = params;
const esClient = await params.esClient;

const esClient = await esClientToResolve;
const installFn = this.getInstallFn({ logger, pluginStop$, tasksTimeoutMs });

// Install component templates in parallel
await Promise.all(
this.componentTemplates.map((componentTemplate) =>
installFn(
createOrUpdateComponentTemplate({
template: componentTemplate,
esClient,
logger,
totalFieldsLimit: this.totalFieldsLimit,
}),
`${componentTemplate.name} component template`
)
)
);
await this.installTemplates(params);

// Install index templates in parallel
await Promise.all(
this.indexTemplates.map((indexTemplate) =>
installFn(
createOrUpdateIndexTemplate({
template: indexTemplate,
esClient,
logger,
}),
`${indexTemplate.name} index template`
)
)
);
const installFn = this.getInstallFn({ logger, pluginStop$, tasksTimeoutMs });

// create data stream when everything is ready
await installFn(
Expand Down
89 changes: 23 additions & 66 deletions packages/kbn-data-stream-adapter/src/data_stream_spaces_adapter.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,59 +7,26 @@
* License v3.0 only", or the "Server Side Public License, v 1".
*/

import { createOrUpdateComponentTemplate } from './create_or_update_component_template';
import { createDataStream, updateDataStreams } from './create_or_update_data_stream';
import { createOrUpdateIndexTemplate } from './create_or_update_index_template';
import {
DataStreamAdapter,
type DataStreamAdapterParams,
IndexPatternAdapter,
type SetIndexTemplateParams,
type InstallParams,
} from './data_stream_adapter';

export class DataStreamSpacesAdapter extends DataStreamAdapter {
private installedSpaceDataStreamName: Map<string, Promise<string>>;
private _installSpace?: (spaceId: string) => Promise<string>;
type InstallIndex,
} from '@kbn/index-adapter';
import { createDataStream, updateDataStreams } from './create_or_update_data_stream';

constructor(private readonly prefix: string, options: DataStreamAdapterParams) {
super(`${prefix}-*`, options); // make indexTemplate `indexPatterns` match all data stream space names
this.installedSpaceDataStreamName = new Map();
export class DataStreamSpacesAdapter extends IndexPatternAdapter {
public setIndexTemplate(params: SetIndexTemplateParams) {
super.setIndexTemplate({ ...params, isDataStream: true });
}

public async install({
logger,
esClient: esClientToResolve,
pluginStop$,
tasksTimeoutMs,
}: InstallParams) {
this.installed = true;
protected async _install(params: InstallParams): Promise<InstallIndex> {
const { logger, pluginStop$, tasksTimeoutMs } = params;

const esClient = await esClientToResolve;
const installFn = this.getInstallFn({ logger, pluginStop$, tasksTimeoutMs });
await this.installTemplates(params);

// Install component templates in parallel
await Promise.all(
this.componentTemplates.map((componentTemplate) =>
installFn(
createOrUpdateComponentTemplate({
template: componentTemplate,
esClient,
logger,
totalFieldsLimit: this.totalFieldsLimit,
}),
`create or update ${componentTemplate.name} component template`
)
)
);

// Install index templates in parallel
await Promise.all(
this.indexTemplates.map((indexTemplate) =>
installFn(
createOrUpdateIndexTemplate({ template: indexTemplate, esClient, logger }),
`create or update ${indexTemplate.name} index template`
)
)
);
const esClient = await params.esClient;
const installFn = this.getInstallFn({ logger, pluginStop$, tasksTimeoutMs });

// Update existing space data streams
await installFn(
Expand All @@ -72,31 +39,21 @@ export class DataStreamSpacesAdapter extends DataStreamAdapter {
`update space data streams`
);

// define function to install data stream for spaces on demand
this._installSpace = async (spaceId: string) => {
const existingInstallPromise = this.installedSpaceDataStreamName.get(spaceId);
if (existingInstallPromise) {
return existingInstallPromise;
}
const name = `${this.prefix}-${spaceId}`;
const installPromise = installFn(
createDataStream({ name, esClient, logger }),
`create ${name} data stream`
).then(() => name);

this.installedSpaceDataStreamName.set(spaceId, installPromise);
return installPromise;
};
// define function to install data stream on demand
return async (name: string) =>
installFn(createDataStream({ name, esClient, logger }), `create ${name} data stream`);
}

/**
* Method to create the data stream for a given space ID.
* It resolves with the full data stream name.
*/
public async installSpace(spaceId: string): Promise<string> {
if (!this._installSpace) {
throw new Error('Cannot installSpace before install');
}
return this._installSpace(spaceId);
await this.createIndex(spaceId);
return this.getIndexName(spaceId);
}

public async getInstalledSpaceName(spaceId: string): Promise<string | undefined> {
return this.installedSpaceDataStreamName.get(spaceId);
return this.getInstalledIndexName(spaceId);
}
}
14 changes: 5 additions & 9 deletions packages/kbn-data-stream-adapter/tsconfig.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,18 +5,14 @@
"types": [
"jest",
"node",
"react",
"@emotion/react/types/css-prop",
"@testing-library/jest-dom",
"@testing-library/react"
]
},
"include": ["**/*.ts", "**/*.tsx"],
"include": ["**/*.ts"],
"kbn_references": [
"@kbn/core",
"@kbn/std",
"@kbn/safer-lodash-set",
"@kbn/logging-mocks",
"@kbn/index-adapter",
],
"exclude": [
"target/**/*"
],
"exclude": ["target/**/*"]
}
Loading

0 comments on commit 953bc0f

Please sign in to comment.