Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Backport 2.x] Standardize source data to input/output transform modals #374

Merged
merged 1 commit into from
Sep 12, 2024
Merged

[Backport 2.x] Standardize source data to input/output transform modals #374

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions common/interfaces.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -394,8 +394,8 @@ export type ModelInputFormField = ModelInput & {
export type ModelOutputFormField = ModelInputFormField;

export type ModelInterface = {
input: ModelInput;
output: ModelOutput;
input?: ModelInput;
output?: ModelOutput;
};

export type ConnectorParameters = {
Expand Down
1 change: 1 addition & 0 deletions public/pages/workflow_detail/workflow_inputs/input_fields/boolean_field.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ export function BooleanField(props: BooleanFieldProps) {
}
onChange={(id) => {
form.setFieldValue(field.name, !field.value);
form.setFieldTouched(field.name, true);
}}
/>
</EuiCompressedFormRow>
Expand Down
48 changes: 30 additions & 18 deletions public/pages/workflow_detail/workflow_inputs/processor_inputs/input_transform_modal.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,9 @@ export function InputTransformModal(props: InputTransformModalProps) {
const dataSourceId = getDataSourceId();
const { values } = useFormikContext<WorkflowFormValues>();

// fetching input data state
const [isFetching, setIsFetching] = useState<boolean>(false);

// source input / transformed output state
const [sourceInput, setSourceInput] = useState<string>('[]');
const [transformedOutput, setTransformedOutput] = useState<string>('{}');
Expand Down Expand Up @@ -116,13 +119,7 @@ export function InputTransformModal(props: InputTransformModalProps) {
) {
let sampleSourceInput = {};
try {
// In the context of ingest or search resp, this input will be an array (list of docs)
// In the context of request, it will be a single JSON
sampleSourceInput =
props.context === PROCESSOR_CONTEXT.INGEST ||
props.context === PROCESSOR_CONTEXT.SEARCH_RESPONSE
? JSON.parse(sourceInput)[0]
: JSON.parse(sourceInput);
sampleSourceInput = JSON.parse(sourceInput);
const output = generateTransform(
sampleSourceInput,
map[selectedOutputOption]
Expand Down Expand Up @@ -167,7 +164,9 @@ export function InputTransformModal(props: InputTransformModalProps) {
<EuiText>Source input</EuiText>
<EuiSmallButton
style={{ width: '100px' }}
isLoading={isFetching}
onClick={async () => {
setIsFetching(true);
switch (props.context) {
case PROCESSOR_CONTEXT.INGEST: {
// get the current ingest pipeline up to, but not including, this processor
Expand Down Expand Up @@ -196,21 +195,31 @@ export function InputTransformModal(props: InputTransformModalProps) {
)
.unwrap()
.then((resp: SimulateIngestPipelineResponse) => {
setSourceInput(unwrapTransformedDocs(resp));
const docObjs = unwrapTransformedDocs(resp);
if (docObjs.length > 0) {
setSourceInput(customStringify(docObjs[0]));
}
})
.catch((error: any) => {
getCore().notifications.toasts.addDanger(
`Failed to fetch input data`
);
})
.finally(() => {
setIsFetching(false);
});
} else {
try {
const docObjs = JSON.parse(
values.ingest.docs
) as {}[];
if (docObjs.length > 0)
setSourceInput(customStringify([docObjs[0]]));
} catch {}
if (docObjs.length > 0) {
setSourceInput(customStringify(docObjs[0]));
}
} catch {
} finally {
setIsFetching(false);
}
}
break;
}
Expand All @@ -230,6 +239,7 @@ export function InputTransformModal(props: InputTransformModalProps) {
if (curSearchPipeline === undefined) {
setSourceInput(values.search.request);
}
setIsFetching(false);
break;
}
case PROCESSOR_CONTEXT.SEARCH_RESPONSE: {
Expand Down Expand Up @@ -257,18 +267,20 @@ export function InputTransformModal(props: InputTransformModalProps) {
)
.unwrap()
.then(async (resp) => {
setSourceInput(
customStringify(
resp.hits.hits.map(
(hit: SearchHit) => hit._source
)
)
const hits = resp.hits.hits.map(
(hit: SearchHit) => hit._source
);
if (hits.length > 0) {
setSourceInput(customStringify(hits[0]));
}
})
.catch((error: any) => {
getCore().notifications.toasts.addDanger(
`Failed to fetch source input data`
);
})
.finally(() => {
setIsFetching(false);
});
break;
}
Expand Down Expand Up @@ -357,7 +369,7 @@ export function InputTransformModal(props: InputTransformModalProps) {
</EuiFlexItem>
)}
<EuiFlexItem grow={true}>
{outputOptions.length === 1 ? (
{outputOptions.length <= 1 ? (
<EuiText>Transformed input</EuiText>
) : (
<EuiCompressedSelect
Expand Down
54 changes: 36 additions & 18 deletions public/pages/workflow_detail/workflow_inputs/processor_inputs/output_transform_modal.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

import React, { useState, useEffect } from 'react';
import { useFormikContext, getIn } from 'formik';
import { cloneDeep, isEmpty, set } from 'lodash';
import { cloneDeep, get, isEmpty, set } from 'lodash';
import {
EuiCodeEditor,
EuiFlexGroup,
Expand Down Expand Up @@ -70,6 +70,9 @@ export function OutputTransformModal(props: OutputTransformModalProps) {
const dataSourceId = getDataSourceId();
const { values } = useFormikContext<WorkflowFormValues>();

// fetching input data state
const [isFetching, setIsFetching] = useState<boolean>(false);

// source input / transformed output state
const [sourceInput, setSourceInput] = useState<string>('[]');
const [transformedOutput, setTransformedOutput] = useState<string>('{}');
Expand All @@ -95,13 +98,7 @@ export function OutputTransformModal(props: OutputTransformModalProps) {
) {
let sampleSourceInput = {};
try {
// In the context of ingest or search resp, this input will be an array (list of docs)
// In the context of request, it will be a single JSON
sampleSourceInput =
props.context === PROCESSOR_CONTEXT.INGEST ||
props.context === PROCESSOR_CONTEXT.SEARCH_RESPONSE
? JSON.parse(sourceInput)[0]
: JSON.parse(sourceInput);
sampleSourceInput = JSON.parse(sourceInput);
const output = generateTransform(
sampleSourceInput,
map[selectedOutputOption]
Expand Down Expand Up @@ -129,7 +126,9 @@ export function OutputTransformModal(props: OutputTransformModalProps) {
<EuiText>Source output</EuiText>
<EuiSmallButton
style={{ width: '100px' }}
isLoading={isFetching}
onClick={async () => {
setIsFetching(true);
switch (props.context) {
// note we skip search request processor context. that is because empty output maps are not supported.
// for more details, see comment in ml_processor_inputs.tsx
Expand Down Expand Up @@ -165,12 +164,24 @@ export function OutputTransformModal(props: OutputTransformModalProps) {
)
.unwrap()
.then((resp: SimulateIngestPipelineResponse) => {
setSourceInput(unwrapTransformedDocs(resp));
try {
const docObjs = unwrapTransformedDocs(resp);
if (docObjs.length > 0) {
const sampleModelResult =
docObjs[0]?.inference_results;
setSourceInput(
customStringify(sampleModelResult)
);
}
} catch {}
})
.catch((error: any) => {
getCore().notifications.toasts.addDanger(
`Failed to fetch input data`
);
})
.finally(() => {
setIsFetching(false);
});
break;
}
Expand Down Expand Up @@ -201,26 +212,33 @@ export function OutputTransformModal(props: OutputTransformModalProps) {
index: values.ingest.index.name,
body: JSON.stringify({
...JSON.parse(values.search.request as string),
search_pipeline: curSearchPipeline,
search_pipeline: curSearchPipeline || {},
}),
},
dataSourceId,
})
)
.unwrap()
.then(async (resp) => {
setSourceInput(
customStringify(
resp.hits.hits.map(
(hit: SearchHit) => hit._source
)
)
);
const hits = resp.hits.hits.map(
(hit: SearchHit) => hit._source
) as any[];
if (hits.length > 0) {
const sampleModelResult = get(
hits,
'0.inference_results.0',
{}
);
setSourceInput(customStringify(sampleModelResult));
}
})
.catch((error: any) => {
getCore().notifications.toasts.addDanger(
`Failed to fetch source output data`
);
})
.finally(() => {
setIsFetching(false);
});
break;
}
Expand Down Expand Up @@ -281,7 +299,7 @@ export function OutputTransformModal(props: OutputTransformModalProps) {
</EuiFlexItem>
<EuiFlexItem>
<>
{outputOptions.length === 1 ? (
{outputOptions.length <= 1 ? (
<EuiText>Transformed output</EuiText>
) : (
<EuiCompressedSelect
Expand Down
7 changes: 2 additions & 5 deletions public/utils/utils.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ export function prepareDocsForSimulate(
// to format them into a more readable string to display
export function unwrapTransformedDocs(
simulatePipelineResponse: SimulateIngestPipelineResponse
) {
): any[] {
let errorDuringSimulate = undefined as string | undefined;
const transformedDocsSources = simulatePipelineResponse.docs.map(
(transformedDoc) => {
Expand All @@ -167,7 +167,7 @@ export function unwrapTransformedDocs(
`Failed to simulate ingest on all documents: ${errorDuringSimulate}`
);
}
return customStringify(transformedDocsSources);
return transformedDocsSources;
}

// ML inference processors will use standard dot notation or JSONPath depending on the input.
Expand All @@ -181,9 +181,6 @@ export function generateTransform(input: {}, map: MapFormValue): {} {
if (mapEntry.value.startsWith(JSONPATH_ROOT_SELECTOR)) {
// JSONPath transform
transformedResult = jsonpath.query(input, path);
// Non-JSONPath bracket notation not supported - throw an error
} else if (mapEntry.value.includes('[') || mapEntry.value.includes(']')) {
throw new Error();
// Standard dot notation
} else {
transformedResult = get(input, path);
Expand Down
14 changes: 12 additions & 2 deletions server/routes/helpers.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,9 @@ import {
MODEL_STATE,
Model,
ModelDict,
ModelInput,
ModelInterface,
ModelOutput,
NO_MODIFICATIONS_FOUND_TEXT,
SearchHit,
WORKFLOW_RESOURCE_TYPE,
Expand Down Expand Up @@ -107,9 +109,17 @@ export function getModelsFromResponses(modelHits: SearchHit[]): ModelDict {
| undefined;
let modelInterface = undefined as ModelInterface | undefined;
if (indexedModelInterface !== undefined) {
let parsedInput = undefined as ModelInput | undefined;
let parsedOutput = undefined as ModelOutput | undefined;
try {
parsedInput = JSON.parse(indexedModelInterface.input);
} catch {}
try {
parsedOutput = JSON.parse(indexedModelInterface.output);
} catch {}
modelInterface = {
input: JSON.parse(indexedModelInterface.input),
output: JSON.parse(indexedModelInterface.output),
input: parsedInput,
output: parsedOutput,
} as ModelInterface;
}

Expand Down
Loading