diff --git a/x-pack/test/api_integration/apis/management/index_management/data_streams.ts b/x-pack/test/api_integration/apis/management/index_management/data_streams.ts index 6ed1ec9ed1c4c..791e23149aff1 100644 --- a/x-pack/test/api_integration/apis/management/index_management/data_streams.ts +++ b/x-pack/test/api_integration/apis/management/index_management/data_streams.ts @@ -9,85 +9,21 @@ import expect from '@kbn/expect'; import { DataStream } from '@kbn/index-management-plugin/common'; import { FtrProviderContext } from '../../../ftr_provider_context'; -// @ts-ignore import { API_BASE_PATH } from './constants'; +import { datastreamsHelpers } from './lib/datastreams.helpers'; export default function ({ getService }: FtrProviderContext) { const supertest = getService('supertest'); - const es = getService('es'); - - const createDataStream = async (name: string) => { - // A data stream requires an index template before it can be created. - await es.indices.putIndexTemplate({ - name, - body: { - // We need to match the names of backing indices with this template. - index_patterns: [name + '*'], - template: { - mappings: { - properties: { - '@timestamp': { - type: 'date', - }, - }, - }, - lifecycle: { - // @ts-expect-error @elastic/elasticsearch enabled prop is not typed yet - enabled: true, - }, - }, - data_stream: {}, - }, - }); - await es.indices.createDataStream({ name }); - }; - - const updateIndexTemplateMappings = async (name: string, mappings: any) => { - await es.indices.putIndexTemplate({ - name, - body: { - // We need to match the names of backing indices with this template. - index_patterns: [name + '*'], - template: { - mappings, - }, - data_stream: {}, - }, - }); - }; - - const getDatastream = async (name: string) => { - const { - data_streams: [datastream], - } = await es.indices.getDataStream({ name }); - return datastream; - }; - - const getMapping = async (name: string) => { - const res = await es.indices.getMapping({ index: name }); - - return Object.values(res)[0]!.mappings; - }; - - const deleteComposableIndexTemplate = async (name: string) => { - await es.indices.deleteIndexTemplate({ name }); - }; - - const deleteDataStream = async (name: string) => { - await es.indices.deleteDataStream({ name }); - await deleteComposableIndexTemplate(name); - }; - - const assertDataStreamStorageSizeExists = (storageSize: string, storageSizeBytes: number) => { - // Storage size of a document doesn't look like it would be deterministic (could vary depending - // on how ES, Lucene, and the file system interact), so we'll just assert its presence and - // type. - expect(storageSize).to.be.ok(); - expect(typeof storageSize).to.be('string'); - expect(storageSizeBytes).to.be.ok(); - expect(typeof storageSizeBytes).to.be('number'); - }; + const { + createDataStream, + deleteDataStream, + assertDataStreamStorageSizeExists, + deleteComposableIndexTemplate, + updateIndexTemplateMappings, + getMapping, + getDatastream, + } = datastreamsHelpers(getService); describe('Data streams', function () { describe('Get', () => { diff --git a/x-pack/test/api_integration/apis/management/index_management/lib/datastreams.helpers.ts b/x-pack/test/api_integration/apis/management/index_management/lib/datastreams.helpers.ts new file mode 100644 index 0000000000000..65e2d733dd696 --- /dev/null +++ b/x-pack/test/api_integration/apis/management/index_management/lib/datastreams.helpers.ts @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; +import { FtrProviderContext } from '../../../../ftr_provider_context'; + +export function datastreamsHelpers(getService: FtrProviderContext['getService']) { + const es = getService('es'); + + const createDataStream = async (name: string) => { + // A data stream requires an index template before it can be created. + await es.indices.putIndexTemplate({ + name, + body: { + // We need to match the names of backing indices with this template. + index_patterns: [name + '*'], + template: { + mappings: { + properties: { + '@timestamp': { + type: 'date', + }, + }, + }, + lifecycle: { + // @ts-expect-error @elastic/elasticsearch enabled prop is not typed yet + enabled: true, + }, + }, + data_stream: {}, + }, + }); + + await es.indices.createDataStream({ name }); + }; + + const updateIndexTemplateMappings = async (name: string, mappings: any) => { + await es.indices.putIndexTemplate({ + name, + body: { + // We need to match the names of backing indices with this template. + index_patterns: [name + '*'], + template: { + mappings, + }, + data_stream: {}, + }, + }); + }; + + const getDatastream = async (name: string) => { + const { + data_streams: [datastream], + } = await es.indices.getDataStream({ name }); + return datastream; + }; + + const getMapping = async (name: string) => { + const res = await es.indices.getMapping({ index: name }); + + return Object.values(res)[0]!.mappings; + }; + + const deleteComposableIndexTemplate = async (name: string) => { + await es.indices.deleteIndexTemplate({ name }); + }; + + const deleteDataStream = async (name: string) => { + await es.indices.deleteDataStream({ name }); + await deleteComposableIndexTemplate(name); + }; + + const assertDataStreamStorageSizeExists = (storageSize: string, storageSizeBytes: number) => { + // Storage size of a document doesn't look like it would be deterministic (could vary depending + // on how ES, Lucene, and the file system interact), so we'll just assert its presence and + // type. + expect(storageSize).to.be.ok(); + expect(typeof storageSize).to.be('string'); + expect(storageSizeBytes).to.be.ok(); + expect(typeof storageSizeBytes).to.be('number'); + }; + + return { + createDataStream, + updateIndexTemplateMappings, + getDatastream, + getMapping, + deleteComposableIndexTemplate, + deleteDataStream, + assertDataStreamStorageSizeExists, + }; +} diff --git a/x-pack/test/api_integration/services/index_management.ts b/x-pack/test/api_integration/services/index_management.ts index 44d33752e4147..f5a57a9b74259 100644 --- a/x-pack/test/api_integration/services/index_management.ts +++ b/x-pack/test/api_integration/services/index_management.ts @@ -9,6 +9,7 @@ import { FtrProviderContext } from '../ftr_provider_context'; import { indicesApi } from '../apis/management/index_management/lib/indices.api'; import { mappingsApi } from '../apis/management/index_management/lib/mappings.api'; import { indicesHelpers } from '../apis/management/index_management/lib/indices.helpers'; +import { datastreamsHelpers } from '../apis/management/index_management/lib/datastreams.helpers'; export function IndexManagementProvider({ getService }: FtrProviderContext) { return { @@ -16,6 +17,9 @@ export function IndexManagementProvider({ getService }: FtrProviderContext) { api: indicesApi(getService), helpers: indicesHelpers(getService), }, + datastreams: { + helpers: datastreamsHelpers(getService), + }, mappings: { api: mappingsApi(getService), }, diff --git a/x-pack/test_serverless/api_integration/test_suites/common/index_management/datastreams.ts b/x-pack/test_serverless/api_integration/test_suites/common/index_management/datastreams.ts new file mode 100644 index 0000000000000..adc84ffecb638 --- /dev/null +++ b/x-pack/test_serverless/api_integration/test_suites/common/index_management/datastreams.ts @@ -0,0 +1,325 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; + +import { DataStream } from '@kbn/index-management-plugin/common'; +import { FtrProviderContext } from '../../../ftr_provider_context'; + +const API_BASE_PATH = '/api/index_management'; + +export default function ({ getService }: FtrProviderContext) { + const supertest = getService('supertest'); + const indexManagementService = getService('indexManagement'); + let helpers: typeof indexManagementService['datastreams']['helpers']; + let createDataStream: typeof helpers['createDataStream']; + let deleteDataStream: typeof helpers['deleteDataStream']; + let deleteComposableIndexTemplate: typeof helpers['deleteComposableIndexTemplate']; + let updateIndexTemplateMappings: typeof helpers['updateIndexTemplateMappings']; + let getMapping: typeof helpers['getMapping']; + let getDatastream: typeof helpers['getDatastream']; + + describe('Data streams', function () { + before(async () => { + ({ + datastreams: { helpers }, + } = indexManagementService); + ({ + createDataStream, + deleteDataStream, + deleteComposableIndexTemplate, + updateIndexTemplateMappings, + getMapping, + getDatastream, + } = helpers); + }); + describe('Get', () => { + const testDataStreamName = 'test-data-stream'; + + before(async () => await createDataStream(testDataStreamName)); + after(async () => await deleteDataStream(testDataStreamName)); + + it('returns an array of data streams', async () => { + const { body: dataStreams } = await supertest + .get(`${API_BASE_PATH}/data_streams`) + .set('kbn-xsrf', 'xxx') + .set('x-elastic-internal-origin', 'xxx') + .expect(200); + + expect(dataStreams).to.be.an('array'); + + // returned array can contain automatically created data streams + const testDataStream = dataStreams.find( + (dataStream: DataStream) => dataStream.name === testDataStreamName + ); + + expect(testDataStream).to.be.ok(); + + // ES determines these values so we'll just echo them back. + const { name: indexName, uuid } = testDataStream!.indices[0]; + + expect(testDataStream).to.eql({ + name: testDataStreamName, + lifecycle: { + enabled: true, + }, + privileges: { + delete_index: true, + manage_data_stream_lifecycle: true, + }, + timeStampField: { name: '@timestamp' }, + indices: [ + { + name: indexName, + uuid, + preferILM: true, + managedBy: 'Data stream lifecycle', + }, + ], + nextGenerationManagedBy: 'Data stream lifecycle', + generation: 1, + health: 'green', + indexTemplateName: testDataStreamName, + hidden: false, + }); + }); + + it('includes stats when provided the includeStats query parameter', async () => { + const { body: dataStreams } = await supertest + .get(`${API_BASE_PATH}/data_streams?includeStats=true`) + .set('kbn-xsrf', 'xxx') + .set('x-elastic-internal-origin', 'xxx') + .expect(200); + + expect(dataStreams).to.be.an('array'); + + // returned array can contain automatically created data streams + const testDataStream = dataStreams.find( + (dataStream: DataStream) => dataStream.name === testDataStreamName + ); + + expect(testDataStream).to.be.ok(); + + // ES determines these values so we'll just echo them back. + const { name: indexName, uuid } = testDataStream!.indices[0]; + const { storageSize, storageSizeBytes, ...dataStreamWithoutStorageSize } = testDataStream!; + + expect(dataStreamWithoutStorageSize).to.eql({ + name: testDataStreamName, + privileges: { + delete_index: true, + manage_data_stream_lifecycle: true, + }, + timeStampField: { name: '@timestamp' }, + indices: [ + { + name: indexName, + managedBy: 'Data stream lifecycle', + preferILM: true, + uuid, + }, + ], + generation: 1, + health: 'green', + indexTemplateName: testDataStreamName, + nextGenerationManagedBy: 'Data stream lifecycle', + maxTimeStamp: 0, + hidden: false, + lifecycle: { + enabled: true, + }, + }); + }); + + it('returns a single data stream by ID', async () => { + const { body: dataStream } = await supertest + .get(`${API_BASE_PATH}/data_streams/${testDataStreamName}`) + .set('kbn-xsrf', 'xxx') + .set('x-elastic-internal-origin', 'xxx') + .expect(200); + + // ES determines these values so we'll just echo them back. + const { name: indexName, uuid } = dataStream.indices[0]; + const { storageSize, storageSizeBytes, ...dataStreamWithoutStorageSize } = dataStream; + + expect(dataStreamWithoutStorageSize).to.eql({ + name: testDataStreamName, + privileges: { + delete_index: true, + manage_data_stream_lifecycle: true, + }, + timeStampField: { name: '@timestamp' }, + indices: [ + { + name: indexName, + managedBy: 'Data stream lifecycle', + preferILM: true, + uuid, + }, + ], + generation: 1, + health: 'green', + indexTemplateName: testDataStreamName, + nextGenerationManagedBy: 'Data stream lifecycle', + maxTimeStamp: 0, + hidden: false, + lifecycle: { + enabled: true, + }, + }); + }); + }); + + describe('Update', () => { + const testDataStreamName = 'test-data-stream'; + + before(async () => await createDataStream(testDataStreamName)); + after(async () => await deleteDataStream(testDataStreamName)); + + it('updates the data retention of a DS', async () => { + const { body } = await supertest + .put(`${API_BASE_PATH}/data_streams/${testDataStreamName}/data_retention`) + .set('kbn-xsrf', 'xxx') + .set('x-elastic-internal-origin', 'xxx') + .send({ + dataRetention: '7d', + }) + .expect(200); + + expect(body).to.eql({ success: true }); + }); + + it('sets data retention to infinite', async () => { + const { body } = await supertest + .put(`${API_BASE_PATH}/data_streams/${testDataStreamName}/data_retention`) + .set('kbn-xsrf', 'xxx') + .set('x-elastic-internal-origin', 'xxx') + .send({}) + .expect(200); + + expect(body).to.eql({ success: true }); + }); + + it('can disable lifecycle for a given policy', async () => { + const { body } = await supertest + .put(`${API_BASE_PATH}/data_streams/${testDataStreamName}/data_retention`) + .set('kbn-xsrf', 'xxx') + .set('x-elastic-internal-origin', 'xxx') + .send({ enabled: false }) + .expect(200); + + expect(body).to.eql({ success: true }); + + const datastream = await getDatastream(testDataStreamName); + expect(datastream.lifecycle).to.be(undefined); + }); + }); + + describe('Delete', () => { + const testDataStreamName1 = 'test-data-stream1'; + const testDataStreamName2 = 'test-data-stream2'; + + before(async () => { + await Promise.all([ + createDataStream(testDataStreamName1), + createDataStream(testDataStreamName2), + ]); + }); + + after(async () => { + // The Delete API only deletes the data streams, so we still need to manually delete their + // related index patterns to clean up. + await Promise.all([ + deleteComposableIndexTemplate(testDataStreamName1), + deleteComposableIndexTemplate(testDataStreamName2), + ]); + }); + + it('deletes multiple data streams', async () => { + await supertest + .post(`${API_BASE_PATH}/delete_data_streams`) + .set('x-elastic-internal-origin', 'xxx') + .set('kbn-xsrf', 'xxx') + .send({ + dataStreams: [testDataStreamName1, testDataStreamName2], + }) + .expect(200); + + await supertest + .get(`${API_BASE_PATH}/data_streams/${testDataStreamName1}`) + .set('kbn-xsrf', 'xxx') + .set('x-elastic-internal-origin', 'xxx') + .expect(404); + + await supertest + .get(`${API_BASE_PATH}/data_streams/${testDataStreamName2}`) + .set('kbn-xsrf', 'xxx') + .set('x-elastic-internal-origin', 'xxx') + .expect(404); + }); + }); + + describe('Mappings from template', () => { + const testDataStreamName1 = 'test-data-stream-mappings-1'; + + before(async () => { + await createDataStream(testDataStreamName1); + }); + + after(async () => { + await deleteDataStream(testDataStreamName1); + }); + + it('Apply mapping from index template', async () => { + const beforeMapping = await getMapping(testDataStreamName1); + expect(beforeMapping.properties).eql({ + '@timestamp': { type: 'date' }, + }); + await updateIndexTemplateMappings(testDataStreamName1, { + properties: { + test: { type: 'integer' }, + }, + }); + await supertest + .post(`${API_BASE_PATH}/data_streams/${testDataStreamName1}/mappings_from_template`) + .set('kbn-xsrf', 'xxx') + .set('x-elastic-internal-origin', 'xxx') + .expect(200); + + const afterMapping = await getMapping(testDataStreamName1); + expect(afterMapping.properties).eql({ + '@timestamp': { type: 'date' }, + test: { type: 'integer' }, + }); + }); + }); + + describe('Rollover', () => { + const testDataStreamName1 = 'test-data-stream-rollover-1'; + + before(async () => { + await createDataStream(testDataStreamName1); + }); + + after(async () => { + await deleteDataStream(testDataStreamName1); + }); + + it('Rollover datastreams', async () => { + await supertest + .post(`${API_BASE_PATH}/data_streams/${testDataStreamName1}/rollover`) + .set('kbn-xsrf', 'xxx') + .set('x-elastic-internal-origin', 'xxx') + .expect(200); + + const datastream = await getDatastream(testDataStreamName1); + + expect(datastream.generation).equal(2); + }); + }); + }); +} diff --git a/x-pack/test_serverless/api_integration/test_suites/common/index_management/index.ts b/x-pack/test_serverless/api_integration/test_suites/common/index_management/index.ts index e06aaf9225cfa..07f5da6aba981 100644 --- a/x-pack/test_serverless/api_integration/test_suites/common/index_management/index.ts +++ b/x-pack/test_serverless/api_integration/test_suites/common/index_management/index.ts @@ -14,6 +14,7 @@ export default function ({ loadTestFile }: FtrProviderContext) { loadTestFile(require.resolve('./index_templates')); loadTestFile(require.resolve('./indices')); loadTestFile(require.resolve('./create_enrich_policies')); + loadTestFile(require.resolve('./datastreams')); loadTestFile(require.resolve('./mappings')); }); } diff --git a/x-pack/test_serverless/tsconfig.json b/x-pack/test_serverless/tsconfig.json index 6f843b935d63f..af6f6e4b25e99 100644 --- a/x-pack/test_serverless/tsconfig.json +++ b/x-pack/test_serverless/tsconfig.json @@ -68,5 +68,6 @@ "@kbn/apm-synthtrace-client", "@kbn/reporting-export-types-csv-common", "@kbn/mock-idp-plugin", + "@kbn/index-management-plugin", ] }