Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[8.x] [IndexAdapter] Extract index-adapter package from data-stream-adapter (#199575) #199848

Merged
merged 2 commits into from
Nov 12, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -567,6 +567,7 @@
"@kbn/i18n-react": "link:packages/kbn-i18n-react",
"@kbn/iframe-embedded-plugin": "link:x-pack/test/functional_embedded/plugins/iframe_embedded",
"@kbn/image-embeddable-plugin": "link:src/plugins/image_embeddable",
"@kbn/index-adapter": "link:packages/kbn-index-adapter",
"@kbn/index-lifecycle-management-plugin": "link:x-pack/plugins/index_lifecycle_management",
"@kbn/index-management-plugin": "link:x-pack/plugins/index_management",
"@kbn/index-management-shared-types": "link:x-pack/packages/index-management/index_management_shared_types",
Expand Down
10 changes: 5 additions & 5 deletions packages/kbn-data-stream-adapter/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,13 @@

export { DataStreamAdapter } from './src/data_stream_adapter';
export { DataStreamSpacesAdapter } from './src/data_stream_spaces_adapter';
export { retryTransientEsErrors } from './src/retry_transient_es_errors';
export { ecsFieldMap, type EcsFieldMap } from './src/field_maps/ecs_field_map';

export { retryTransientEsErrors, ecsFieldMap } from '@kbn/index-adapter';
export type {
DataStreamAdapterParams,
SetComponentTemplateParams,
SetIndexTemplateParams,
InstallParams,
} from './src/data_stream_adapter';
export * from './src/field_maps/types';
EcsFieldMap,
} from '@kbn/index-adapter';

export * from '@kbn/index-adapter/src/field_maps/types';
5 changes: 3 additions & 2 deletions packages/kbn-data-stream-adapter/kibana.jsonc
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
{
"type": "shared-common",
"type": "shared-server",
"id": "@kbn/data-stream-adapter",
"owner": "@elastic/security-threat-hunting-explore"
"owner": "@elastic/security-threat-hunting",
"visibility": "shared"
}
Original file line number Diff line number Diff line change
Expand Up @@ -136,10 +136,11 @@ describe('createOrUpdateDataStream', () => {
it(`should create data stream if not exists`, async () => {
esClient.indices.getDataStream.mockResolvedValueOnce({ data_streams: [] });

await createDataStream({
await createOrUpdateDataStream({
esClient,
logger,
name,
totalFieldsLimit,
});

expect(esClient.indices.createDataStream).toHaveBeenCalledWith({ name });
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import type { IndicesDataStream } from '@elastic/elasticsearch/lib/api/types';
import type { IndicesSimulateIndexTemplateResponse } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
import type { Logger, ElasticsearchClient } from '@kbn/core/server';
import { get } from 'lodash';
import { retryTransientEsErrors } from './retry_transient_es_errors';
import { retryTransientEsErrors } from '@kbn/index-adapter';

interface UpdateIndexMappingsOpts {
logger: Logger;
Expand Down Expand Up @@ -168,7 +168,7 @@ export async function createDataStream({
esClient,
name,
}: CreateDataStreamParams): Promise<void> {
logger.info(`Creating data stream - ${name}`);
logger.debug(`Checking data stream exists - ${name}`);

// check if data stream exists
let dataStreamExists = false;
Expand All @@ -189,6 +189,7 @@ export async function createDataStream({
if (dataStreamExists) {
return;
}
logger.info(`Installing data stream - ${name}`);

try {
await retryTransientEsErrors(() => esClient.indices.createDataStream({ name }), { logger });
Expand Down
139 changes: 8 additions & 131 deletions packages/kbn-data-stream-adapter/src/data_stream_adapter.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,145 +7,22 @@
* License v3.0 only", or the "Server Side Public License, v 1".
*/

import type {
ClusterPutComponentTemplateRequest,
IndicesIndexSettings,
IndicesPutIndexTemplateIndexTemplateMapping,
IndicesPutIndexTemplateRequest,
} from '@elastic/elasticsearch/lib/api/types';
import type { Logger, ElasticsearchClient } from '@kbn/core/server';
import type { Subject } from 'rxjs';
import type { FieldMap } from './field_maps/types';
import { createOrUpdateComponentTemplate } from './create_or_update_component_template';
import { IndexAdapter, SetIndexTemplateParams, type InstallParams } from '@kbn/index-adapter';
import { createOrUpdateDataStream } from './create_or_update_data_stream';
import { createOrUpdateIndexTemplate } from './create_or_update_index_template';
import { InstallShutdownError, installWithTimeout } from './install_with_timeout';
import { getComponentTemplate, getIndexTemplate } from './resource_installer_utils';

export interface DataStreamAdapterParams {
kibanaVersion: string;
totalFieldsLimit?: number;
}
export interface SetComponentTemplateParams {
name: string;
fieldMap: FieldMap;
settings?: IndicesIndexSettings;
dynamic?: 'strict' | boolean;
}
export interface SetIndexTemplateParams {
name: string;
componentTemplateRefs?: string[];
namespace?: string;
template?: IndicesPutIndexTemplateIndexTemplateMapping;
hidden?: boolean;
}

export interface GetInstallFnParams {
logger: Logger;
pluginStop$: Subject<void>;
tasksTimeoutMs?: number;
}
export interface InstallParams {
logger: Logger;
esClient: ElasticsearchClient | Promise<ElasticsearchClient>;
pluginStop$: Subject<void>;
tasksTimeoutMs?: number;
}

const DEFAULT_FIELDS_LIMIT = 2500;

export class DataStreamAdapter {
protected readonly kibanaVersion: string;
protected readonly totalFieldsLimit: number;
protected componentTemplates: ClusterPutComponentTemplateRequest[] = [];
protected indexTemplates: IndicesPutIndexTemplateRequest[] = [];
protected installed: boolean;

constructor(protected readonly name: string, options: DataStreamAdapterParams) {
this.installed = false;
this.kibanaVersion = options.kibanaVersion;
this.totalFieldsLimit = options.totalFieldsLimit ?? DEFAULT_FIELDS_LIMIT;
}

public setComponentTemplate(params: SetComponentTemplateParams) {
if (this.installed) {
throw new Error('Cannot set component template after install');
}
this.componentTemplates.push(getComponentTemplate(params));
}

export class DataStreamAdapter extends IndexAdapter {
public setIndexTemplate(params: SetIndexTemplateParams) {
if (this.installed) {
throw new Error('Cannot set index template after install');
}
this.indexTemplates.push(
getIndexTemplate({
...params,
indexPatterns: [this.name],
kibanaVersion: this.kibanaVersion,
totalFieldsLimit: this.totalFieldsLimit,
})
);
}

protected getInstallFn({ logger, pluginStop$, tasksTimeoutMs }: GetInstallFnParams) {
return async (promise: Promise<void>, description?: string): Promise<void> => {
try {
await installWithTimeout({
installFn: () => promise,
description,
timeoutMs: tasksTimeoutMs,
pluginStop$,
});
} catch (err) {
if (err instanceof InstallShutdownError) {
logger.info(err.message);
} else {
throw err;
}
}
};
super.setIndexTemplate({ ...params, isDataStream: true });
}

public async install({
logger,
esClient: esClientToResolve,
pluginStop$,
tasksTimeoutMs,
}: InstallParams) {
public async install(params: InstallParams) {
this.installed = true;
const { logger, pluginStop$, tasksTimeoutMs } = params;
const esClient = await params.esClient;

const esClient = await esClientToResolve;
const installFn = this.getInstallFn({ logger, pluginStop$, tasksTimeoutMs });

// Install component templates in parallel
await Promise.all(
this.componentTemplates.map((componentTemplate) =>
installFn(
createOrUpdateComponentTemplate({
template: componentTemplate,
esClient,
logger,
totalFieldsLimit: this.totalFieldsLimit,
}),
`${componentTemplate.name} component template`
)
)
);
await this.installTemplates(params);

// Install index templates in parallel
await Promise.all(
this.indexTemplates.map((indexTemplate) =>
installFn(
createOrUpdateIndexTemplate({
template: indexTemplate,
esClient,
logger,
}),
`${indexTemplate.name} index template`
)
)
);
const installFn = this.getInstallFn({ logger, pluginStop$, tasksTimeoutMs });

// create data stream when everything is ready
await installFn(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,59 +7,26 @@
* License v3.0 only", or the "Server Side Public License, v 1".
*/

import { createOrUpdateComponentTemplate } from './create_or_update_component_template';
import { createDataStream, updateDataStreams } from './create_or_update_data_stream';
import { createOrUpdateIndexTemplate } from './create_or_update_index_template';
import {
DataStreamAdapter,
type DataStreamAdapterParams,
IndexPatternAdapter,
type SetIndexTemplateParams,
type InstallParams,
} from './data_stream_adapter';

export class DataStreamSpacesAdapter extends DataStreamAdapter {
private installedSpaceDataStreamName: Map<string, Promise<string>>;
private _installSpace?: (spaceId: string) => Promise<string>;
type InstallIndex,
} from '@kbn/index-adapter';
import { createDataStream, updateDataStreams } from './create_or_update_data_stream';

constructor(private readonly prefix: string, options: DataStreamAdapterParams) {
super(`${prefix}-*`, options); // make indexTemplate `indexPatterns` match all data stream space names
this.installedSpaceDataStreamName = new Map();
export class DataStreamSpacesAdapter extends IndexPatternAdapter {
public setIndexTemplate(params: SetIndexTemplateParams) {
super.setIndexTemplate({ ...params, isDataStream: true });
}

public async install({
logger,
esClient: esClientToResolve,
pluginStop$,
tasksTimeoutMs,
}: InstallParams) {
this.installed = true;
protected async _install(params: InstallParams): Promise<InstallIndex> {
const { logger, pluginStop$, tasksTimeoutMs } = params;

const esClient = await esClientToResolve;
const installFn = this.getInstallFn({ logger, pluginStop$, tasksTimeoutMs });
await this.installTemplates(params);

// Install component templates in parallel
await Promise.all(
this.componentTemplates.map((componentTemplate) =>
installFn(
createOrUpdateComponentTemplate({
template: componentTemplate,
esClient,
logger,
totalFieldsLimit: this.totalFieldsLimit,
}),
`create or update ${componentTemplate.name} component template`
)
)
);

// Install index templates in parallel
await Promise.all(
this.indexTemplates.map((indexTemplate) =>
installFn(
createOrUpdateIndexTemplate({ template: indexTemplate, esClient, logger }),
`create or update ${indexTemplate.name} index template`
)
)
);
const esClient = await params.esClient;
const installFn = this.getInstallFn({ logger, pluginStop$, tasksTimeoutMs });

// Update existing space data streams
await installFn(
Expand All @@ -72,31 +39,21 @@ export class DataStreamSpacesAdapter extends DataStreamAdapter {
`update space data streams`
);

// define function to install data stream for spaces on demand
this._installSpace = async (spaceId: string) => {
const existingInstallPromise = this.installedSpaceDataStreamName.get(spaceId);
if (existingInstallPromise) {
return existingInstallPromise;
}
const name = `${this.prefix}-${spaceId}`;
const installPromise = installFn(
createDataStream({ name, esClient, logger }),
`create ${name} data stream`
).then(() => name);

this.installedSpaceDataStreamName.set(spaceId, installPromise);
return installPromise;
};
// define function to install data stream on demand
return async (name: string) =>
installFn(createDataStream({ name, esClient, logger }), `create ${name} data stream`);
}

/**
* Method to create the data stream for a given space ID.
* It resolves with the full data stream name.
*/
public async installSpace(spaceId: string): Promise<string> {
if (!this._installSpace) {
throw new Error('Cannot installSpace before install');
}
return this._installSpace(spaceId);
await this.createIndex(spaceId);
return this.getIndexName(spaceId);
}

public async getInstalledSpaceName(spaceId: string): Promise<string | undefined> {
return this.installedSpaceDataStreamName.get(spaceId);
return this.getInstalledIndexName(spaceId);
}
}
14 changes: 5 additions & 9 deletions packages/kbn-data-stream-adapter/tsconfig.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,18 +5,14 @@
"types": [
"jest",
"node",
"react",
"@emotion/react/types/css-prop",
"@testing-library/jest-dom",
"@testing-library/react"
]
},
"include": ["**/*.ts", "**/*.tsx"],
"include": ["**/*.ts"],
"kbn_references": [
"@kbn/core",
"@kbn/std",
"@kbn/safer-lodash-set",
"@kbn/logging-mocks",
"@kbn/index-adapter",
],
"exclude": [
"target/**/*"
],
"exclude": ["target/**/*"]
}
Loading