Skip to content

Commit

Permalink
[Obs AI Assistant] Add retry statements as an attempt to resolve flak…
Browse files Browse the repository at this point in the history
…y tests (elastic#200022)

Closes elastic#192222

## Summary

### Problem
The test appears to be flaky, potentially because the entries are not
available at the time of retrieval. This cannot be reproduced locally or
via the flaky test runner. (more details
[here](elastic#196026 (comment)))

### Solution
Add a retry when fetching the instructions and check whether the number
of instructions returned by the API endpoint is the same number of
instructions expected.

### Checklist

- [x] [Flaky Test
Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was
used on any tests changed
  • Loading branch information
viduni94 authored and wayneseymour committed Nov 18, 2024
1 parent 1ebb244 commit aebe405
Showing 1 changed file with 56 additions and 48 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ export default function ApiTest({ getService }: FtrProviderContext) {
const es = getService('es');
const ml = getService('ml');
const log = getService('log');
const retry = getService('retry');
const getScopedApiClientForUsername = getService('getScopedApiClientForUsername');

describe('Knowledge base user instructions', () => {
Expand Down Expand Up @@ -94,62 +95,69 @@ export default function ApiTest({ getService }: FtrProviderContext) {
});

it('"editor" can retrieve their own private instructions and the public instruction', async () => {
const res = await observabilityAIAssistantAPIClient.editor({
endpoint: 'GET /internal/observability_ai_assistant/kb/user_instructions',
});
await retry.try(async () => {
const res = await observabilityAIAssistantAPIClient.editor({
endpoint: 'GET /internal/observability_ai_assistant/kb/user_instructions',
});

const instructions = res.body.userInstructions;
const instructions = res.body.userInstructions;
expect(instructions).to.have.length(3);

const sortById = (data: Array<Instruction & { public?: boolean }>) => sortBy(data, 'id');
const sortById = (data: Array<Instruction & { public?: boolean }>) => sortBy(data, 'id');

expect(sortById(instructions)).to.eql(
sortById([
{
id: 'private-doc-from-editor',
public: false,
text: 'Private user instruction from "editor"',
},
{
id: 'public-doc-from-editor',
public: true,
text: 'Public user instruction from "editor"',
},
{
id: 'public-doc-from-secondary_editor',
public: true,
text: 'Public user instruction from "secondary_editor"',
},
])
);
expect(sortById(instructions)).to.eql(
sortById([
{
id: 'private-doc-from-editor',
public: false,
text: 'Private user instruction from "editor"',
},
{
id: 'public-doc-from-editor',
public: true,
text: 'Public user instruction from "editor"',
},
{
id: 'public-doc-from-secondary_editor',
public: true,
text: 'Public user instruction from "secondary_editor"',
},
])
);
});
});

it('"secondaryEditor" can retrieve their own private instructions and the public instruction', async () => {
const res = await observabilityAIAssistantAPIClient.secondaryEditor({
endpoint: 'GET /internal/observability_ai_assistant/kb/user_instructions',
});
const instructions = res.body.userInstructions;
await retry.try(async () => {
const res = await observabilityAIAssistantAPIClient.secondaryEditor({
endpoint: 'GET /internal/observability_ai_assistant/kb/user_instructions',
});

const sortById = (data: Array<Instruction & { public?: boolean }>) => sortBy(data, 'id');
const instructions = res.body.userInstructions;
expect(instructions).to.have.length(3);

expect(sortById(instructions)).to.eql(
sortById([
{
id: 'public-doc-from-editor',
public: true,
text: 'Public user instruction from "editor"',
},
{
id: 'public-doc-from-secondary_editor',
public: true,
text: 'Public user instruction from "secondary_editor"',
},
{
id: 'private-doc-from-secondary_editor',
public: false,
text: 'Private user instruction from "secondary_editor"',
},
])
);
const sortById = (data: Array<Instruction & { public?: boolean }>) => sortBy(data, 'id');

expect(sortById(instructions)).to.eql(
sortById([
{
id: 'public-doc-from-editor',
public: true,
text: 'Public user instruction from "editor"',
},
{
id: 'public-doc-from-secondary_editor',
public: true,
text: 'Public user instruction from "secondary_editor"',
},
{
id: 'private-doc-from-secondary_editor',
public: false,
text: 'Private user instruction from "secondary_editor"',
},
])
);
});
});
});

Expand Down

0 comments on commit aebe405

Please sign in to comment.