-
Notifications
You must be signed in to change notification settings - Fork 8.3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Browse files
Browse the repository at this point in the history
> [!Warning] > The following files were updated as a result of merge conflicts that should be reviewed > - `/x-pack/test_serverless/api_integration/test_suites/security/config.ts` > - `/x-pack/test_serverless/functional/test_suites/search/config.ts` > - `/x-pack/test_serverless/tsconfig.json` # Backport This will backport the following commits from `main` to `8.x`: - [[Data Usage] setup integration tests (#197112)](#197112) <!--- Backport version: 8.9.8 --> ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sqren/backport) <!--BACKPORT [{"author":{"name":"Sandra G","email":"[email protected]"},"sourceCommit":{"committedDate":"2024-11-06T18:20:49Z","message":"[Data Usage] setup integration tests (#197112)\n\n## Summary\r\n\r\nAdds serverless api integration tests and basic functional smoke test\r\nfor data usage plugin\r\n\r\nBoth are skipped in mki until `xpack.dataUsage.enabled` is enabled by\r\ndefault in serverless\r\n\r\n### `GET /internal/api/data_usage/data_streams`\r\n\r\n### `POST /internal/api/data_usage/metrics`\r\n- skipped in MKI because we'll need to make sure real credentials are\r\nbeing used via the `xpack.dataUsage.autoops*`\r\n- we start a mock server at localhost:9000 and set that the config\r\n(`xpack.dataUsage.autoops.api.url=http://localhost:9000'`) along with\r\nfake credentials for the other `xpack.dataUsage.autoops*` values. If\r\nwe're not in MKI these values will be used and the mock server will\r\nrespond to the request at `http://localhost:9000`. If we are in MKI, the\r\nreal values and credentials should be set, otherwise it will fail as\r\nthese kibana config values in the tests are not passed into the MKI\r\nenvironment.\r\n\r\n---------\r\n\r\nCo-authored-by: kibanamachine <[email protected]>","sha":"4f9ab47291362185a113755eefe2583df5f9827b","branchLabelMapping":{"^v9.0.0$":"main","^v8.17.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:skip","v9.0.0","backport:version","v8.17.0"],"number":197112,"url":"https://github.com/elastic/kibana/pull/197112","mergeCommit":{"message":"[Data Usage] setup integration tests (#197112)\n\n## Summary\r\n\r\nAdds serverless api integration tests and basic functional smoke test\r\nfor data usage plugin\r\n\r\nBoth are skipped in mki until `xpack.dataUsage.enabled` is enabled by\r\ndefault in serverless\r\n\r\n### `GET /internal/api/data_usage/data_streams`\r\n\r\n### `POST /internal/api/data_usage/metrics`\r\n- skipped in MKI because we'll need to make sure real credentials are\r\nbeing used via the `xpack.dataUsage.autoops*`\r\n- we start a mock server at localhost:9000 and set that the config\r\n(`xpack.dataUsage.autoops.api.url=http://localhost:9000'`) along with\r\nfake credentials for the other `xpack.dataUsage.autoops*` values. If\r\nwe're not in MKI these values will be used and the mock server will\r\nrespond to the request at `http://localhost:9000`. If we are in MKI, the\r\nreal values and credentials should be set, otherwise it will fail as\r\nthese kibana config values in the tests are not passed into the MKI\r\nenvironment.\r\n\r\n---------\r\n\r\nCo-authored-by: kibanamachine <[email protected]>","sha":"4f9ab47291362185a113755eefe2583df5f9827b"}},"sourceBranch":"main","suggestedTargetBranches":["8.x"],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","labelRegex":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/197112","number":197112,"mergeCommit":{"message":"[Data Usage] setup integration tests (#197112)\n\n## Summary\r\n\r\nAdds serverless api integration tests and basic functional smoke test\r\nfor data usage plugin\r\n\r\nBoth are skipped in mki until `xpack.dataUsage.enabled` is enabled by\r\ndefault in serverless\r\n\r\n### `GET /internal/api/data_usage/data_streams`\r\n\r\n### `POST /internal/api/data_usage/metrics`\r\n- skipped in MKI because we'll need to make sure real credentials are\r\nbeing used via the `xpack.dataUsage.autoops*`\r\n- we start a mock server at localhost:9000 and set that the config\r\n(`xpack.dataUsage.autoops.api.url=http://localhost:9000'`) along with\r\nfake credentials for the other `xpack.dataUsage.autoops*` values. If\r\nwe're not in MKI these values will be used and the mock server will\r\nrespond to the request at `http://localhost:9000`. If we are in MKI, the\r\nreal values and credentials should be set, otherwise it will fail as\r\nthese kibana config values in the tests are not passed into the MKI\r\nenvironment.\r\n\r\n---------\r\n\r\nCo-authored-by: kibanamachine <[email protected]>","sha":"4f9ab47291362185a113755eefe2583df5f9827b"}},{"branch":"8.x","label":"v8.17.0","labelRegex":"^v8.17.0$","isSourceBranch":false,"state":"NOT_CREATED"}]}] BACKPORT--> --------- Signed-off-by: Ash <[email protected]> Co-authored-by: Sandra G <[email protected]> Co-authored-by: kibanamachine <[email protected]>
- Loading branch information
1 parent
f7c8827
commit c4a5ebf
Showing
25 changed files
with
392 additions
and
13 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
17 changes: 17 additions & 0 deletions
17
x-pack/test_serverless/api_integration/test_suites/common/data_usage/index.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,17 @@ | ||
/* | ||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
* or more contributor license agreements. Licensed under the Elastic License | ||
* 2.0; you may not use this file except in compliance with the Elastic License | ||
* 2.0. | ||
*/ | ||
|
||
import { FtrProviderContext } from '../../../ftr_provider_context'; | ||
|
||
export default function ({ loadTestFile }: FtrProviderContext) { | ||
describe('Serverless Data Usage APIs', function () { | ||
this.tags(['esGate']); | ||
|
||
loadTestFile(require.resolve('./tests/data_streams')); | ||
loadTestFile(require.resolve('./tests/metrics')); | ||
}); | ||
} |
23 changes: 23 additions & 0 deletions
23
x-pack/test_serverless/api_integration/test_suites/common/data_usage/mock_api.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,23 @@ | ||
/* | ||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
* or more contributor license agreements. Licensed under the Elastic License | ||
* 2.0; you may not use this file except in compliance with the Elastic License | ||
* 2.0. | ||
*/ | ||
|
||
import { createServer } from '@mswjs/http-middleware'; | ||
import { UsageMetricsAutoOpsResponseSchemaBody } from '@kbn/data-usage-plugin/common/rest_types'; | ||
import { http, HttpResponse, StrictResponse } from 'msw'; | ||
import { mockAutoOpsResponse } from './mock_data'; | ||
|
||
export const setupMockServer = () => { | ||
const server = createServer(autoOpsHandler); | ||
return server; | ||
}; | ||
|
||
const autoOpsHandler = http.post( | ||
'/', | ||
async ({ request }): Promise<StrictResponse<UsageMetricsAutoOpsResponseSchemaBody>> => { | ||
return HttpResponse.json(mockAutoOpsResponse); | ||
} | ||
); |
43 changes: 43 additions & 0 deletions
43
x-pack/test_serverless/api_integration/test_suites/common/data_usage/mock_data.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,43 @@ | ||
/* | ||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
* or more contributor license agreements. Licensed under the Elastic License | ||
* 2.0; you may not use this file except in compliance with the Elastic License | ||
* 2.0. | ||
*/ | ||
|
||
export const mockAutoOpsResponse = { | ||
metrics: { | ||
ingest_rate: [ | ||
{ | ||
name: 'metrics-system.cpu-default', | ||
data: [ | ||
[1726858530000, 13756849], | ||
[1726862130000, 14657904], | ||
], | ||
}, | ||
{ | ||
name: 'logs-nginx.access-default', | ||
data: [ | ||
[1726858530000, 12894623], | ||
[1726862130000, 14436905], | ||
], | ||
}, | ||
], | ||
storage_retained: [ | ||
{ | ||
name: 'metrics-system.cpu-default', | ||
data: [ | ||
[1726858530000, 12576413], | ||
[1726862130000, 13956423], | ||
], | ||
}, | ||
{ | ||
name: 'logs-nginx.access-default', | ||
data: [ | ||
[1726858530000, 12894623], | ||
[1726862130000, 14436905], | ||
], | ||
}, | ||
], | ||
}, | ||
}; |
56 changes: 56 additions & 0 deletions
56
x-pack/test_serverless/api_integration/test_suites/common/data_usage/tests/data_streams.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,56 @@ | ||
/* | ||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
* or more contributor license agreements. Licensed under the Elastic License | ||
* 2.0; you may not use this file except in compliance with the Elastic License | ||
* 2.0. | ||
*/ | ||
|
||
import expect from '@kbn/expect'; | ||
import { SupertestWithRoleScope } from '@kbn/test-suites-xpack/api_integration/deployment_agnostic/services/role_scoped_supertest'; | ||
import { DataStreamsResponseBodySchemaBody } from '@kbn/data-usage-plugin/common/rest_types'; | ||
import { DATA_USAGE_DATA_STREAMS_API_ROUTE } from '@kbn/data-usage-plugin/common'; | ||
import { FtrProviderContext } from '../../../../ftr_provider_context'; | ||
|
||
export default function ({ getService }: FtrProviderContext) { | ||
const svlDatastreamsHelpers = getService('svlDatastreamsHelpers'); | ||
const roleScopedSupertest = getService('roleScopedSupertest'); | ||
let supertestAdminWithCookieCredentials: SupertestWithRoleScope; | ||
const testDataStreamName = 'test-data-stream'; | ||
describe(`GET ${DATA_USAGE_DATA_STREAMS_API_ROUTE}`, function () { | ||
// due to the plugin depending on yml config (xpack.dataUsage.enabled), we cannot test in MKI until it is on by default | ||
this.tags(['skipMKI']); | ||
before(async () => { | ||
await svlDatastreamsHelpers.createDataStream(testDataStreamName); | ||
supertestAdminWithCookieCredentials = await roleScopedSupertest.getSupertestWithRoleScope( | ||
'admin', | ||
{ | ||
useCookieHeader: true, | ||
withInternalHeaders: true, | ||
} | ||
); | ||
}); | ||
after(async () => { | ||
await svlDatastreamsHelpers.deleteDataStream(testDataStreamName); | ||
}); | ||
|
||
it('returns created data streams', async () => { | ||
const res = await supertestAdminWithCookieCredentials | ||
.get(DATA_USAGE_DATA_STREAMS_API_ROUTE) | ||
.set('elastic-api-version', '1'); | ||
const dataStreams: DataStreamsResponseBodySchemaBody = res.body; | ||
const foundStream = dataStreams.find((stream) => stream.name === testDataStreamName); | ||
expect(foundStream?.name).to.be(testDataStreamName); | ||
expect(foundStream?.storageSizeBytes).to.be(0); | ||
expect(res.statusCode).to.be(200); | ||
}); | ||
it('returns system indices', async () => { | ||
const res = await supertestAdminWithCookieCredentials | ||
.get(DATA_USAGE_DATA_STREAMS_API_ROUTE) | ||
.set('elastic-api-version', '1'); | ||
const dataStreams: DataStreamsResponseBodySchemaBody = res.body; | ||
const systemDataStreams = dataStreams.filter((stream) => stream.name.startsWith('.')); | ||
expect(systemDataStreams.length).to.be.greaterThan(0); | ||
expect(res.statusCode).to.be(200); | ||
}); | ||
}); | ||
} |
110 changes: 110 additions & 0 deletions
110
x-pack/test_serverless/api_integration/test_suites/common/data_usage/tests/metrics.ts
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,110 @@ | ||
/* | ||
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
* or more contributor license agreements. Licensed under the Elastic License | ||
* 2.0; you may not use this file except in compliance with the Elastic License | ||
* 2.0. | ||
*/ | ||
|
||
import expect from '@kbn/expect'; | ||
import http from 'http'; | ||
|
||
import { SupertestWithRoleScope } from '@kbn/test-suites-xpack/api_integration/deployment_agnostic/services/role_scoped_supertest'; | ||
import { UsageMetricsRequestBody } from '@kbn/data-usage-plugin/common/rest_types'; | ||
import { DATA_USAGE_METRICS_API_ROUTE } from '@kbn/data-usage-plugin/common'; | ||
import { transformMetricsData } from '@kbn/data-usage-plugin/server/routes/internal/usage_metrics_handler'; | ||
import { FtrProviderContext } from '../../../../ftr_provider_context'; | ||
import { setupMockServer } from '../mock_api'; | ||
import { mockAutoOpsResponse } from '../mock_data'; | ||
|
||
export default function ({ getService }: FtrProviderContext) { | ||
const svlDatastreamsHelpers = getService('svlDatastreamsHelpers'); | ||
const roleScopedSupertest = getService('roleScopedSupertest'); | ||
let supertestAdminWithCookieCredentials: SupertestWithRoleScope; | ||
const mockAutoopsApiService = setupMockServer(); | ||
describe('Metrics', function () { | ||
let mockApiServer: http.Server; | ||
// due to the plugin depending on yml config (xpack.dataUsage.enabled), we cannot test in MKI until it is on by default | ||
this.tags(['skipMKI']); | ||
|
||
before(async () => { | ||
mockApiServer = mockAutoopsApiService.listen(9000); | ||
supertestAdminWithCookieCredentials = await roleScopedSupertest.getSupertestWithRoleScope( | ||
'admin', | ||
{ | ||
useCookieHeader: true, | ||
withInternalHeaders: true, | ||
} | ||
); | ||
}); | ||
|
||
after(() => { | ||
mockApiServer.close(); | ||
}); | ||
describe(`POST ${DATA_USAGE_METRICS_API_ROUTE}`, () => { | ||
const testDataStreamName = 'test-data-stream'; | ||
before(async () => await svlDatastreamsHelpers.createDataStream(testDataStreamName)); | ||
after(async () => await svlDatastreamsHelpers.deleteDataStream(testDataStreamName)); | ||
it('returns 400 with non-existent data streams', async () => { | ||
const requestBody: UsageMetricsRequestBody = { | ||
from: 'now-24h/h', | ||
to: 'now', | ||
metricTypes: ['ingest_rate', 'storage_retained'], | ||
dataStreams: ['invalid-data-stream'], | ||
}; | ||
const res = await supertestAdminWithCookieCredentials | ||
.post(DATA_USAGE_METRICS_API_ROUTE) | ||
.set('elastic-api-version', '1') | ||
.send(requestBody); | ||
expect(res.statusCode).to.be(400); | ||
expect(res.body.message).to.be('Failed to retrieve data streams'); | ||
}); | ||
|
||
it('returns 400 when requesting no data streams', async () => { | ||
const requestBody = { | ||
from: 'now-24h/h', | ||
to: 'now', | ||
metricTypes: ['ingest_rate'], | ||
dataStreams: [], | ||
}; | ||
const res = await supertestAdminWithCookieCredentials | ||
.post(DATA_USAGE_METRICS_API_ROUTE) | ||
.set('elastic-api-version', '1') | ||
.send(requestBody); | ||
expect(res.statusCode).to.be(400); | ||
expect(res.body.message).to.be('[request body.dataStreams]: no data streams selected'); | ||
}); | ||
|
||
it('returns 400 when requesting an invalid metric type', async () => { | ||
const requestBody = { | ||
from: 'now-24h/h', | ||
to: 'now', | ||
metricTypes: [testDataStreamName], | ||
dataStreams: ['datastream'], | ||
}; | ||
const res = await supertestAdminWithCookieCredentials | ||
.post(DATA_USAGE_METRICS_API_ROUTE) | ||
.set('elastic-api-version', '1') | ||
.send(requestBody); | ||
expect(res.statusCode).to.be(400); | ||
expect(res.body.message).to.be( | ||
'[request body.metricTypes]: must be one of ingest_rate, storage_retained, search_vcu, ingest_vcu, ml_vcu, index_latency, index_rate, search_latency, search_rate' | ||
); | ||
}); | ||
|
||
it('returns 200 with valid request', async () => { | ||
const requestBody: UsageMetricsRequestBody = { | ||
from: 'now-24h/h', | ||
to: 'now', | ||
metricTypes: ['ingest_rate', 'storage_retained'], | ||
dataStreams: [testDataStreamName], | ||
}; | ||
const res = await supertestAdminWithCookieCredentials | ||
.post(DATA_USAGE_METRICS_API_ROUTE) | ||
.set('elastic-api-version', '1') | ||
.send(requestBody); | ||
expect(res.statusCode).to.be(200); | ||
expect(res.body).to.eql(transformMetricsData(mockAutoOpsResponse)); | ||
}); | ||
}); | ||
}); | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.