Skip to content

Commit

Permalink
[8.x] [Obs AI Assistant] Add retry statements as an attempt to resolv…
Browse files Browse the repository at this point in the history 
…e flaky tests (#200022) (#200213)

# Backport

This will backport the following commits from `main` to `8.x`:
- [[Obs AI Assistant] Add retry statements as an attempt to resolve
flaky tests (#200022)](#200022)

<!--- Backport version: 9.4.3 -->

### Questions ?
Please refer to the [Backport tool
documentation](https://github.com/sqren/backport)

<!--BACKPORT [{"author":{"name":"Viduni
Wickramarachchi","email":"[email protected]"},"sourceCommit":{"committedDate":"2024-11-14T16:09:42Z","message":"[Obs
AI Assistant] Add retry statements as an attempt to resolve flaky tests
(#200022)\n\nCloses
https://github.com/elastic/kibana/issues/192222\r\n\r\n##
Summary\r\n\r\n### Problem\r\nThe test appears to be flaky, potentially
because the entries are not\r\navailable at the time of retrieval. This
cannot be reproduced locally or\r\nvia the flaky test runner. (more
details\r\n[here](https://github.com/elastic/kibana/pull/196026#issuecomment-2409056856))\r\n\r\n###
Solution\r\nAdd a retry when fetching the instructions and check whether
the number\r\nof instructions returned by the API endpoint is the same
number of\r\ninstructions expected.\r\n\r\n### Checklist\r\n\r\n- [x]
[Flaky
Test\r\nRunner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1)
was\r\nused on any tests
changed","sha":"53c05a33e7fe917b7d64b7d3ca664aaf3f9cf3ca","branchLabelMapping":{"^v9.0.0$":"main","^v8.17.0$":"8.x","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:enhancement","v9.0.0","Team:Obs
AI
Assistant","ci:project-deploy-observability","backport:version","v8.17.0"],"title":"[Obs
AI Assistant] Add retry statements as an attempt to resolve flaky
tests","number":200022,"url":"https://github.com/elastic/kibana/pull/200022","mergeCommit":{"message":"[Obs
AI Assistant] Add retry statements as an attempt to resolve flaky tests
(#200022)\n\nCloses
https://github.com/elastic/kibana/issues/192222\r\n\r\n##
Summary\r\n\r\n### Problem\r\nThe test appears to be flaky, potentially
because the entries are not\r\navailable at the time of retrieval. This
cannot be reproduced locally or\r\nvia the flaky test runner. (more
details\r\n[here](https://github.com/elastic/kibana/pull/196026#issuecomment-2409056856))\r\n\r\n###
Solution\r\nAdd a retry when fetching the instructions and check whether
the number\r\nof instructions returned by the API endpoint is the same
number of\r\ninstructions expected.\r\n\r\n### Checklist\r\n\r\n- [x]
[Flaky
Test\r\nRunner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1)
was\r\nused on any tests
changed","sha":"53c05a33e7fe917b7d64b7d3ca664aaf3f9cf3ca"}},"sourceBranch":"main","suggestedTargetBranches":["8.x"],"targetPullRequestStates":[{"branch":"main","label":"v9.0.0","branchLabelMappingKey":"^v9.0.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/200022","number":200022,"mergeCommit":{"message":"[Obs
AI Assistant] Add retry statements as an attempt to resolve flaky tests
(#200022)\n\nCloses
https://github.com/elastic/kibana/issues/192222\r\n\r\n##
Summary\r\n\r\n### Problem\r\nThe test appears to be flaky, potentially
because the entries are not\r\navailable at the time of retrieval. This
cannot be reproduced locally or\r\nvia the flaky test runner. (more
details\r\n[here](https://github.com/elastic/kibana/pull/196026#issuecomment-2409056856))\r\n\r\n###
Solution\r\nAdd a retry when fetching the instructions and check whether
the number\r\nof instructions returned by the API endpoint is the same
number of\r\ninstructions expected.\r\n\r\n### Checklist\r\n\r\n- [x]
[Flaky
Test\r\nRunner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1)
was\r\nused on any tests
changed","sha":"53c05a33e7fe917b7d64b7d3ca664aaf3f9cf3ca"}},{"branch":"8.x","label":"v8.17.0","branchLabelMappingKey":"^v8.17.0$","isSourceBranch":false,"state":"NOT_CREATED"}]}]
BACKPORT-->

Co-authored-by: Viduni Wickramarachchi <[email protected]>
  • Loading branch information
@kibanamachine @viduni94
kibanamachine and viduni94 authored Nov 14, 2024
1 parent 356fc8f commit 61354f0
Showing 1 changed file with 56 additions and 48 deletions.
104 changes: 56 additions & 48 deletions ...i_assistant_api_integration/tests/knowledge_base/knowledge_base_user_instructions.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
const es = getService('es');
const ml = getService('ml');
const log = getService('log');
const retry = getService('retry');
const getScopedApiClientForUsername = getService('getScopedApiClientForUsername');

describe('Knowledge base user instructions', () => {
Expand Down Expand Up @@ -85,62 +86,69 @@ export default function ApiTest({ getService }: FtrProviderContext) {
});

it('"editor" can retrieve their own private instructions and the public instruction', async () => {
const res = await observabilityAIAssistantAPIClient.editor({
endpoint: 'GET /internal/observability_ai_assistant/kb/user_instructions',
});
await retry.try(async () => {
const res = await observabilityAIAssistantAPIClient.editor({
endpoint: 'GET /internal/observability_ai_assistant/kb/user_instructions',
});

const instructions = res.body.userInstructions;
const instructions = res.body.userInstructions;
expect(instructions).to.have.length(3);

const sortById = (data: Array<Instruction & { public?: boolean }>) => sortBy(data, 'id');
const sortById = (data: Array<Instruction & { public?: boolean }>) => sortBy(data, 'id');

expect(sortById(instructions)).to.eql(
sortById([
{
id: 'private-doc-from-editor',
public: false,
text: 'Private user instruction from "editor"',
},
{
id: 'public-doc-from-editor',
public: true,
text: 'Public user instruction from "editor"',
},
{
id: 'public-doc-from-secondary_editor',
public: true,
text: 'Public user instruction from "secondary_editor"',
},
])
);
expect(sortById(instructions)).to.eql(
sortById([
{
id: 'private-doc-from-editor',
public: false,
text: 'Private user instruction from "editor"',
},
{
id: 'public-doc-from-editor',
public: true,
text: 'Public user instruction from "editor"',
},
{
id: 'public-doc-from-secondary_editor',
public: true,
text: 'Public user instruction from "secondary_editor"',
},
])
);
});
});

it('"secondaryEditor" can retrieve their own private instructions and the public instruction', async () => {
const res = await observabilityAIAssistantAPIClient.secondaryEditor({
endpoint: 'GET /internal/observability_ai_assistant/kb/user_instructions',
});
const instructions = res.body.userInstructions;
await retry.try(async () => {
const res = await observabilityAIAssistantAPIClient.secondaryEditor({
endpoint: 'GET /internal/observability_ai_assistant/kb/user_instructions',
});

const sortById = (data: Array<Instruction & { public?: boolean }>) => sortBy(data, 'id');
const instructions = res.body.userInstructions;
expect(instructions).to.have.length(3);

expect(sortById(instructions)).to.eql(
sortById([
{
id: 'public-doc-from-editor',
public: true,
text: 'Public user instruction from "editor"',
},
{
id: 'public-doc-from-secondary_editor',
public: true,
text: 'Public user instruction from "secondary_editor"',
},
{
id: 'private-doc-from-secondary_editor',
public: false,
text: 'Private user instruction from "secondary_editor"',
},
])
);
const sortById = (data: Array<Instruction & { public?: boolean }>) => sortBy(data, 'id');

expect(sortById(instructions)).to.eql(
sortById([
{
id: 'public-doc-from-editor',
public: true,
text: 'Public user instruction from "editor"',
},
{
id: 'public-doc-from-secondary_editor',
public: true,
text: 'Public user instruction from "secondary_editor"',
},
{
id: 'private-doc-from-secondary_editor',
public: false,
text: 'Private user instruction from "secondary_editor"',
},
])
);
});
});
});

Expand Down

0 comments on commit 61354f0

Please sign in to comment.