diff --git a/.changeset/clean-toes-learn.md b/.changeset/clean-toes-learn.md new file mode 100644 index 0000000000..7b87f06420 --- /dev/null +++ b/.changeset/clean-toes-learn.md @@ -0,0 +1,4 @@ +--- +'@finos/legend-graph': patch +'@finos/legend-application-studio': patch +--- diff --git a/.changeset/kind-rabbits-draw.md b/.changeset/kind-rabbits-draw.md new file mode 100644 index 0000000000..457d23317c --- /dev/null +++ b/.changeset/kind-rabbits-draw.md @@ -0,0 +1,6 @@ +--- +'@finos/legend-extension-dsl-data-space': minor +'@finos/legend-application-query': minor +--- + +Use PMCD returned by mapping analysis to build minimal graph for query diff --git a/packages/legend-application-query/src/components/__test-utils__/QueryEditorComponentTestUtils.tsx b/packages/legend-application-query/src/components/__test-utils__/QueryEditorComponentTestUtils.tsx index e8f1e769e5..e7a2f9024c 100644 --- a/packages/legend-application-query/src/components/__test-utils__/QueryEditorComponentTestUtils.tsx +++ b/packages/legend-application-query/src/components/__test-utils__/QueryEditorComponentTestUtils.tsx @@ -22,7 +22,6 @@ import { Query, LightQuery, RawLambda, - PackageableElementExplicitReference, type RawMappingModelCoverageAnalysisResult, } from '@finos/legend-graph'; import { DepotServerClient } from '@finos/legend-server-depot'; @@ -128,10 +127,8 @@ export const TEST__setUpQueryEditor = async ( query.owner = lightQuery.owner; query.isCurrentUserQuery = lightQuery.isCurrentUserQuery; const _mapping = graphManagerState.graph.getMapping(mappingPath); - query.mapping = PackageableElementExplicitReference.create(_mapping); - query.runtime = PackageableElementExplicitReference.create( - graphManagerState.graph.getRuntime(runtimePath), - ); + query.mapping = mappingPath; + query.runtime = runtimePath; query.content = 'some content'; createSpy( diff --git a/packages/legend-application-query/src/stores/LegendQueryApplicationPlugin.ts b/packages/legend-application-query/src/stores/LegendQueryApplicationPlugin.ts index 1e90584b97..f8d127a771 100644 --- a/packages/legend-application-query/src/stores/LegendQueryApplicationPlugin.ts +++ b/packages/legend-application-query/src/stores/LegendQueryApplicationPlugin.ts @@ -15,8 +15,8 @@ */ import { LegendApplicationPlugin } from '@finos/legend-application'; -import type { Query } from '@finos/legend-graph'; import type { QueryBuilderState } from '@finos/legend-query-builder'; +import type { GeneratorFn } from '@finos/legend-shared'; import type React from 'react'; import type { LegendQueryPluginManager } from '../application/LegendQueryPluginManager.js'; import type { @@ -24,6 +24,7 @@ import type { QueryEditorStore, } from './QueryEditorStore.js'; import type { QuerySetupLandingPageStore } from './QuerySetupStore.js'; +import type { Query } from '@finos/legend-graph'; export enum QuerySetupActionTag { PRODUCTIONIZATION = 'Productionization', @@ -56,6 +57,10 @@ export type ExistingQueryEditorStateBuilder = ( editorStore: ExistingQueryEditorStore, ) => Promise; +export type QueryGraphBuilderGetter = ( + editorStore: QueryEditorStore, +) => ((editorStore: QueryEditorStore) => GeneratorFn) | undefined; + export type QueryEditorActionConfiguration = { key: string; renderer: ( @@ -98,6 +103,11 @@ export abstract class LegendQueryApplicationPlugin extends LegendApplicationPlug */ getExtraExistingQueryEditorStateBuilders?(): ExistingQueryEditorStateBuilder[]; + /** + * Get the list of query graph builders + */ + getExtraQueryGraphBuilderGetters?(): QueryGraphBuilderGetter[]; + /** * Get the list of query editor action renderer configurations. */ diff --git a/packages/legend-application-query/src/stores/QueryEditorStore.ts b/packages/legend-application-query/src/stores/QueryEditorStore.ts index 1e0416e09e..27edf88c24 100644 --- a/packages/legend-application-query/src/stores/QueryEditorStore.ts +++ b/packages/legend-application-query/src/stores/QueryEditorStore.ts @@ -293,6 +293,7 @@ export abstract class QueryEditorStore { setExistingQueryName: action, initialize: flow, buildGraph: flow, + buildFullGraph: flow, searchExistingQueryName: flow, }); @@ -381,6 +382,10 @@ export abstract class QueryEditorStore { // do nothing } + requiresGraphBuilding(): boolean { + return true; + } + async buildQueryForPersistence( query: Query, rawLambda: RawLambda, @@ -401,10 +406,9 @@ export abstract class QueryEditorStore { RuntimePointer, 'Query runtime must be of type runtime pointer', ); - query.mapping = PackageableElementExplicitReference.create( - this.queryBuilderState.executionContextState.mapping, - ); - query.runtime = runtimeValue.packageableRuntime; + (query.mapping = + this.queryBuilderState.executionContextState.mapping.path), + (query.runtime = runtimeValue.packageableRuntime.value.path); query.content = await this.graphManagerState.graphManager.lambdaToPureCode(rawLambda); config?.decorator?.(query); @@ -486,6 +490,7 @@ export abstract class QueryEditorStore { yield this.setUpEditorState(); yield flowResult(this.buildGraph()); + this.queryBuilderState = (yield this.initializeQueryBuilderState( stopWatch, )) as QueryBuilderState; @@ -537,7 +542,7 @@ export abstract class QueryEditorStore { ); } - *buildGraph(): GeneratorFn { + *buildFullGraph(): GeneratorFn { const stopWatch = new StopWatch(); const { groupId, artifactId, versionId } = this.getProjectInfo(); @@ -626,6 +631,20 @@ export abstract class QueryEditorStore { graphBuilderReportData, ); } + + *buildGraph(): GeneratorFn { + const queryGraphBuilderGetters = this.applicationStore.pluginManager + .getApplicationPlugins() + .flatMap((plugin) => plugin.getExtraQueryGraphBuilderGetters?.() ?? []); + for (const getter of queryGraphBuilderGetters) { + const builderFunction = getter(this); + if (builderFunction) { + yield flowResult(builderFunction(this)); + return; + } + } + yield flowResult(this.buildFullGraph()); + } } export class MappingQueryCreatorStore extends QueryEditorStore { @@ -986,23 +1005,22 @@ export class ExistingQueryEditorStore extends QueryEditorStore { } override async setUpEditorState(): Promise { - this.setLightQuery( - await this.graphManagerState.graphManager.getLightQuery(this.queryId), - ); - } - - async initializeQueryBuilderState( - stopWatch: StopWatch, - ): Promise { const query = await this.graphManagerState.graphManager.getQuery( this.queryId, this.graphManagerState.graph, ); this.setQuery(query); + this.setLightQuery(toLightQuery(query)); LegendQueryUserDataHelper.addRecentlyViewedQuery( this.applicationStore.userDataService, query.id, ); + } + + async initializeQueryBuilderState( + stopWatch: StopWatch, + ): Promise { + const query = guaranteeNonNullable(this.query); let queryBuilderState: QueryBuilderState | undefined; const existingQueryEditorStateBuilders = this.applicationStore.pluginManager .getApplicationPlugins() @@ -1025,11 +1043,11 @@ export class ExistingQueryEditorStore extends QueryEditorStore { this.applicationStore.config.options.queryBuilderConfig, ); - queryBuilderState.executionContextState.setMapping(query.mapping.value); + const mapping = this.graphManagerState.graph.getMapping(query.mapping); + const runtime = this.graphManagerState.graph.getRuntime(query.runtime); + queryBuilderState.executionContextState.setMapping(mapping); queryBuilderState.executionContextState.setRuntimeValue( - new RuntimePointer( - PackageableElementExplicitReference.create(query.runtime.value), - ), + new RuntimePointer(PackageableElementExplicitReference.create(runtime)), ); // leverage initialization of query builder state to ensure we handle unsupported queries diff --git a/packages/legend-application-studio/src/components/editor/editor-group/FunctionEditor.tsx b/packages/legend-application-studio/src/components/editor/editor-group/FunctionEditor.tsx index e5e7e47587..e107d39dbe 100644 --- a/packages/legend-application-studio/src/components/editor/editor-group/FunctionEditor.tsx +++ b/packages/legend-application-studio/src/components/editor/editor-group/FunctionEditor.tsx @@ -1105,8 +1105,7 @@ export const FunctionEditor = observer(() => { ); await flowResult( embeddedQueryBuilderState.setEmbeddedQueryBuilderConfiguration({ - setupQueryBuilderState: (): QueryBuilderState => - functionQueryBuilderState, + setupQueryBuilderState: async () => functionQueryBuilderState, actionConfigs: [ { key: 'save-query-btn', diff --git a/packages/legend-application-studio/src/components/editor/editor-group/mapping-editor/DEPRECATED__MappingTestEditor.tsx b/packages/legend-application-studio/src/components/editor/editor-group/mapping-editor/DEPRECATED__MappingTestEditor.tsx index f56f46469f..f4f9328fb6 100644 --- a/packages/legend-application-studio/src/components/editor/editor-group/mapping-editor/DEPRECATED__MappingTestEditor.tsx +++ b/packages/legend-application-studio/src/components/editor/editor-group/mapping-editor/DEPRECATED__MappingTestEditor.tsx @@ -112,7 +112,7 @@ const MappingTestQueryEditor = observer( const embeddedQueryBuilderState = editorStore.embeddedQueryBuilderState; await flowResult( embeddedQueryBuilderState.setEmbeddedQueryBuilderConfiguration({ - setupQueryBuilderState: (): QueryBuilderState => { + setupQueryBuilderState: async () => { const queryBuilderState = new MappingExecutionQueryBuilderState( embeddedQueryBuilderState.editorStore.applicationStore, embeddedQueryBuilderState.editorStore.graphManagerState, diff --git a/packages/legend-application-studio/src/components/editor/editor-group/mapping-editor/MappingExecutionBuilder.tsx b/packages/legend-application-studio/src/components/editor/editor-group/mapping-editor/MappingExecutionBuilder.tsx index 092f2fc345..6355d15cff 100644 --- a/packages/legend-application-studio/src/components/editor/editor-group/mapping-editor/MappingExecutionBuilder.tsx +++ b/packages/legend-application-studio/src/components/editor/editor-group/mapping-editor/MappingExecutionBuilder.tsx @@ -205,7 +205,7 @@ const MappingExecutionQueryEditor = observer( const embeddedQueryBuilderState = editorStore.embeddedQueryBuilderState; await flowResult( embeddedQueryBuilderState.setEmbeddedQueryBuilderConfiguration({ - setupQueryBuilderState: (): QueryBuilderState => { + setupQueryBuilderState: async () => { const queryBuilderState = new MappingExecutionQueryBuilderState( embeddedQueryBuilderState.editorStore.applicationStore, embeddedQueryBuilderState.editorStore.graphManagerState, diff --git a/packages/legend-application-studio/src/components/editor/editor-group/mapping-editor/MappingTestableEditor.tsx b/packages/legend-application-studio/src/components/editor/editor-group/mapping-editor/MappingTestableEditor.tsx index bea58ebc0c..b369c79373 100644 --- a/packages/legend-application-studio/src/components/editor/editor-group/mapping-editor/MappingTestableEditor.tsx +++ b/packages/legend-application-studio/src/components/editor/editor-group/mapping-editor/MappingTestableEditor.tsx @@ -589,7 +589,7 @@ const MappingTestSuiteQueryEditor = observer( const embeddedQueryBuilderState = editorStore.embeddedQueryBuilderState; await flowResult( embeddedQueryBuilderState.setEmbeddedQueryBuilderConfiguration({ - setupQueryBuilderState: (): QueryBuilderState => { + setupQueryBuilderState: async () => { const queryBuilderState = new MappingExecutionQueryBuilderState( embeddedQueryBuilderState.editorStore.applicationStore, embeddedQueryBuilderState.editorStore.graphManagerState, diff --git a/packages/legend-application-studio/src/components/editor/editor-group/service-editor/ServiceExecutionQueryEditor.tsx b/packages/legend-application-studio/src/components/editor/editor-group/service-editor/ServiceExecutionQueryEditor.tsx index 60aeb28758..2cc4ccb740 100644 --- a/packages/legend-application-studio/src/components/editor/editor-group/service-editor/ServiceExecutionQueryEditor.tsx +++ b/packages/legend-application-studio/src/components/editor/editor-group/service-editor/ServiceExecutionQueryEditor.tsx @@ -125,7 +125,7 @@ export const ServiceExecutionQueryEditor = observer( executionState.selectedExecutionContextState; await flowResult( embeddedQueryBuilderState.setEmbeddedQueryBuilderConfiguration({ - setupQueryBuilderState: (): QueryBuilderState => { + setupQueryBuilderState: async () => { const queryBuilderState = new ServiceQueryBuilderState( embeddedQueryBuilderState.editorStore.applicationStore, embeddedQueryBuilderState.editorStore.graphManagerState, @@ -459,7 +459,7 @@ export const queryService = async ( : undefined; await flowResult( embeddedQueryBuilderState.setEmbeddedQueryBuilderConfiguration({ - setupQueryBuilderState: (): QueryBuilderState => { + setupQueryBuilderState: async () => { const queryBuilderState = new ServiceQueryBuilderState( embeddedQueryBuilderState.editorStore.applicationStore, embeddedQueryBuilderState.editorStore.graphManagerState, diff --git a/packages/legend-application-studio/src/components/editor/editor-group/uml-editor/ClassQueryBuilder.tsx b/packages/legend-application-studio/src/components/editor/editor-group/uml-editor/ClassQueryBuilder.tsx index ccd8c83326..6bf11b1058 100644 --- a/packages/legend-application-studio/src/components/editor/editor-group/uml-editor/ClassQueryBuilder.tsx +++ b/packages/legend-application-studio/src/components/editor/editor-group/uml-editor/ClassQueryBuilder.tsx @@ -375,7 +375,7 @@ export const queryClass = async ( const embeddedQueryBuilderState = editorStore.embeddedQueryBuilderState; await flowResult( embeddedQueryBuilderState.setEmbeddedQueryBuilderConfiguration({ - setupQueryBuilderState: () => { + setupQueryBuilderState: async () => { const queryBuilderState = new ClassQueryBuilderState( embeddedQueryBuilderState.editorStore.applicationStore, embeddedQueryBuilderState.editorStore.graphManagerState, diff --git a/packages/legend-application-studio/src/stores/editor/EmbeddedQueryBuilderState.ts b/packages/legend-application-studio/src/stores/editor/EmbeddedQueryBuilderState.ts index 03581d7bd7..7b5f5ee2eb 100644 --- a/packages/legend-application-studio/src/stores/editor/EmbeddedQueryBuilderState.ts +++ b/packages/legend-application-studio/src/stores/editor/EmbeddedQueryBuilderState.ts @@ -30,7 +30,7 @@ type EmbeddedQueryBuilderActionConfiguration = { }; type EmbeddedQueryBuilderConfiguration = { - setupQueryBuilderState: () => QueryBuilderState; + setupQueryBuilderState: () => Promise; disableCompile?: boolean | undefined; actionConfigs: EmbeddedQueryBuilderActionConfiguration[]; }; @@ -100,7 +100,8 @@ export class EmbeddedQueryBuilderState { } } if (!this.editorStore.graphState.error) { - this.queryBuilderState = config.setupQueryBuilderState(); + this.queryBuilderState = + (yield config.setupQueryBuilderState()) as QueryBuilderState; this.actionConfigs = config.actionConfigs; this.editorStore.applicationStore.layoutService.setBackdropContainerElementID( QUERY_BUILDER_COMPONENT_ELEMENT_ID.BACKDROP_CONTAINER, diff --git a/packages/legend-extension-dsl-data-space/src/components/query/DSL_DataSpace_LegendQueryApplicationPlugin.tsx b/packages/legend-extension-dsl-data-space/src/components/query/DSL_DataSpace_LegendQueryApplicationPlugin.tsx index c969f796ee..677b749ae1 100644 --- a/packages/legend-extension-dsl-data-space/src/components/query/DSL_DataSpace_LegendQueryApplicationPlugin.tsx +++ b/packages/legend-extension-dsl-data-space/src/components/query/DSL_DataSpace_LegendQueryApplicationPlugin.tsx @@ -17,14 +17,16 @@ import packageJson from '../../../package.json' assert { type: 'json' }; import { type QuerySetupActionConfiguration, - type ExistingQueryEditorStateBuilder, - type ExistingQueryEditorStore, + ExistingQueryEditorStore, LegendQueryApplicationPlugin, generateExistingQueryEditorRoute, LEGEND_QUERY_APP_EVENT, LegendQueryEventHelper, createViewProjectHandler, createViewSDLCProjectHandler, + type ExistingQueryEditorStateBuilder, + type QueryGraphBuilderGetter, + type QueryEditorStore, } from '@finos/legend-application-query'; import { SquareIcon } from '@finos/legend-art'; import { @@ -38,23 +40,43 @@ import { } from '../../__lib__/query/DSL_DataSpace_LegendQueryNavigation.js'; import { DataSpaceQueryCreator } from './DataSpaceQueryCreator.js'; import { createQueryDataSpaceTaggedValue } from '../../stores/query/DataSpaceQueryCreatorStore.js'; -import { Query, isValidFullPath } from '@finos/legend-graph'; +import { + Query, + isValidFullPath, + GRAPH_MANAGER_EVENT, + createGraphBuilderReport, +} from '@finos/legend-graph'; import { QUERY_PROFILE_PATH, QUERY_PROFILE_TAG_DATA_SPACE, } from '../../graph/DSL_DataSpace_MetaModelConst.js'; import { - DataSpaceQueryBuilderState, DataSpaceProjectInfo, + DataSpaceQueryBuilderState, } from '../../stores/query/DataSpaceQueryBuilderState.js'; import type { DataSpaceInfo } from '../../stores/query/DataSpaceInfo.js'; import { getOwnDataSpace } from '../../graph-manager/DSL_DataSpace_GraphManagerHelper.js'; -import { assertErrorThrown, LogEvent, uuid } from '@finos/legend-shared'; +import { + assertErrorThrown, + isString, + LogEvent, + StopWatch, + uuid, + type GeneratorFn, +} from '@finos/legend-shared'; import type { QueryBuilderState } from '@finos/legend-query-builder'; import { DataSpaceQuerySetup } from './DataSpaceQuerySetup.js'; import { DSL_DataSpace_getGraphManagerExtension } from '../../graph-manager/protocol/pure/DSL_DataSpace_PureGraphManagerExtension.js'; -import { StoreProjectData } from '@finos/legend-server-depot'; +import { + retrieveProjectEntitiesWithDependencies, + StoreProjectData, +} from '@finos/legend-server-depot'; import { retrieveAnalyticsResultCache } from '../../graph-manager/action/analytics/DataSpaceAnalysisHelper.js'; +import { flowResult } from 'mobx'; + +function* buildGraph(): GeneratorFn { + // do nothing +} export class DSL_DataSpace_LegendQueryApplicationPlugin extends LegendQueryApplicationPlugin { constructor() { @@ -110,173 +132,256 @@ export class DSL_DataSpace_LegendQueryApplicationPlugin extends LegendQueryAppli taggedValue.tag === QUERY_PROFILE_TAG_DATA_SPACE && isValidFullPath(taggedValue.value), ); - + let isLightGraphEnabled = true; if (dataSpaceTaggedValue) { const dataSpacePath = dataSpaceTaggedValue.value; - const dataSpace = getOwnDataSpace( - dataSpacePath, - editorStore.graphManagerState.graph, - ); - const mapping = query.mapping.value; - const matchingExecutionContext = dataSpace.executionContexts.find( - (ec) => ec.mapping.value === mapping, - ); - if (!matchingExecutionContext) { - // if a matching execution context is not found, it means this query is not - // properly created from a data space, therefore, we cannot support this case - return undefined; - } + const mappingPath = query.mapping; + const { groupId, artifactId, versionId } = + editorStore.getProjectInfo(); let dataSpaceAnalysisResult; - try { - const project = StoreProjectData.serialization.fromJson( - await editorStore.depotServerClient.getProject( - query.groupId, - query.artifactId, - ), - ); - dataSpaceAnalysisResult = - await DSL_DataSpace_getGraphManagerExtension( - editorStore.graphManagerState.graphManager, - ).retrieveDataSpaceAnalysisFromCache(() => - retrieveAnalyticsResultCache( - project, - query.versionId, - dataSpace.path, - editorStore.depotServerClient, + if (dataSpacePath && isString(mappingPath)) { + try { + editorStore.initState.setMessage( + 'Fetching dataspace analysis result', + ); + const project = StoreProjectData.serialization.fromJson( + await editorStore.depotServerClient.getProject( + groupId, + artifactId, ), ); - } catch { - // do nothing - } - const projectInfo = new DataSpaceProjectInfo( - query.groupId, - query.artifactId, - query.versionId, - createViewProjectHandler(editorStore.applicationStore), - createViewSDLCProjectHandler( + const graph_buildReport = createGraphBuilderReport(); + const stopWatch = new StopWatch(); + // initialize system + stopWatch.record(); + await editorStore.graphManagerState.initializeSystem(); + stopWatch.record( + GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH_SYSTEM__SUCCESS, + ); + const dependency_buildReport = createGraphBuilderReport(); + dataSpaceAnalysisResult = + await DSL_DataSpace_getGraphManagerExtension( + editorStore.graphManagerState.graphManager, + ).analyzeDataSpaceCoverage( + dataSpacePath, + () => + retrieveProjectEntitiesWithDependencies( + project, + versionId, + editorStore.depotServerClient, + ), + () => + retrieveAnalyticsResultCache( + project, + versionId, + dataSpacePath, + editorStore.depotServerClient, + ), + undefined, + graph_buildReport, + editorStore.graphManagerState.graph, + undefined, + mappingPath, + editorStore.getProjectInfo(), + editorStore.applicationStore.notificationService, + ); + + // report + stopWatch.record(GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH__SUCCESS); + const graphBuilderReportData = { + timings: + editorStore.applicationStore.timeService.finalizeTimingsRecord( + stopWatch, + ), + dependencies: dependency_buildReport, + dependenciesCount: + editorStore.graphManagerState.graph.dependencyManager + .numberOfDependencies, + graph: graph_buildReport, + }; + editorStore.logBuildGraphMetrics(graphBuilderReportData); + + editorStore.applicationStore.logService.info( + LogEvent.create(GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH__SUCCESS), + graphBuilderReportData, + ); + } catch (error) { + editorStore.applicationStore.logService.error( + LogEvent.create(LEGEND_QUERY_APP_EVENT.GENERIC_FAILURE), + error, + ); + isLightGraphEnabled = false; + editorStore.graphManagerState.graph = + editorStore.graphManagerState.createNewGraph(); + await flowResult(editorStore.buildFullGraph()); + } + + const dataSpace = getOwnDataSpace( + dataSpacePath, + editorStore.graphManagerState.graph, + ); + const mapping = + editorStore.graphManagerState.graph.getMapping(mappingPath); + const matchingExecutionContext = dataSpace.executionContexts.find( + (ec) => ec.mapping.value === mapping, + ); + if (!matchingExecutionContext) { + // if a matching execution context is not found, it means this query is not + // properly created from a data space, therefore, we cannot support this case + return undefined; + } + const projectInfo = new DataSpaceProjectInfo( + query.groupId, + query.artifactId, + query.versionId, + createViewProjectHandler(editorStore.applicationStore), + createViewSDLCProjectHandler( + editorStore.applicationStore, + editorStore.depotServerClient, + ), + ); + const dataSpaceQueryBuilderState = new DataSpaceQueryBuilderState( editorStore.applicationStore, + editorStore.graphManagerState, editorStore.depotServerClient, - ), - ); - const dataSpaceQueryBuilderState = new DataSpaceQueryBuilderState( - editorStore.applicationStore, - editorStore.graphManagerState, - editorStore.depotServerClient, - dataSpace, - matchingExecutionContext, - (dataSpaceInfo: DataSpaceInfo) => { - if (dataSpaceInfo.defaultExecutionContext) { - const createQuery = async (): Promise => { - // prepare the new query to save - const _query = new Query(); - _query.name = query.name; - _query.id = query.id; - _query.groupId = query.groupId; - _query.artifactId = query.artifactId; - _query.versionId = query.versionId; - _query.mapping = query.mapping; - _query.runtime = query.runtime; - _query.taggedValues = [ - createQueryDataSpaceTaggedValue(dataSpaceInfo.path), - ].concat( - (query.taggedValues ?? []).filter( - (taggedValue) => taggedValue !== dataSpaceTaggedValue, - ), - ); - _query.stereotypes = query.stereotypes; - _query.content = query.content; - _query.owner = query.owner; - _query.lastUpdatedAt = query.lastUpdatedAt; + dataSpace, + matchingExecutionContext, + isLightGraphEnabled, + async (dataSpaceInfo: DataSpaceInfo) => { + if (dataSpaceInfo.defaultExecutionContext) { + const createQuery = async (): Promise => { + // prepare the new query to save + const _query = new Query(); + _query.name = query.name; + _query.id = query.id; + _query.groupId = query.groupId; + _query.artifactId = query.artifactId; + _query.versionId = query.versionId; + _query.mapping = query.mapping; + _query.runtime = query.runtime; + _query.taggedValues = [ + createQueryDataSpaceTaggedValue(dataSpaceInfo.path), + ].concat( + (query.taggedValues ?? []).filter( + (taggedValue) => taggedValue !== dataSpaceTaggedValue, + ), + ); + _query.stereotypes = query.stereotypes; + _query.content = query.content; + _query.owner = query.owner; + _query.lastUpdatedAt = query.lastUpdatedAt; - try { - if (!query.isCurrentUserQuery) { - _query.id = uuid(); - const newQuery = - await editorStore.graphManagerState.graphManager.createQuery( + try { + if (!query.isCurrentUserQuery) { + _query.id = uuid(); + const newQuery = + await editorStore.graphManagerState.graphManager.createQuery( + _query, + editorStore.graphManagerState.graph, + ); + editorStore.applicationStore.notificationService.notifySuccess( + `Successfully created query!`, + ); + LegendQueryEventHelper.notify_QueryCreateSucceeded( + editorStore.applicationStore.eventService, + { queryId: newQuery.id }, + ); + editorStore.applicationStore.navigationService.navigator.goToLocation( + generateExistingQueryEditorRoute(newQuery.id), + ); + } else { + await editorStore.graphManagerState.graphManager.updateQuery( _query, editorStore.graphManagerState.graph, ); - editorStore.applicationStore.notificationService.notifySuccess( - `Successfully created query!`, - ); - LegendQueryEventHelper.notify_QueryCreateSucceeded( - editorStore.applicationStore.eventService, - { queryId: newQuery.id }, - ); - editorStore.applicationStore.navigationService.navigator.goToLocation( - generateExistingQueryEditorRoute(newQuery.id), - ); - } else { - await editorStore.graphManagerState.graphManager.updateQuery( - _query, - editorStore.graphManagerState.graph, + editorStore.applicationStore.notificationService.notifySuccess( + `Successfully updated query!`, + ); + editorStore.applicationStore.navigationService.navigator.reload(); + } + } catch (error) { + assertErrorThrown(error); + editorStore.applicationStore.logService.error( + LogEvent.create(LEGEND_QUERY_APP_EVENT.GENERIC_FAILURE), + error, ); - editorStore.applicationStore.notificationService.notifySuccess( - `Successfully updated query!`, + editorStore.applicationStore.notificationService.notifyError( + error, ); - editorStore.applicationStore.navigationService.navigator.reload(); } - } catch (error) { - assertErrorThrown(error); - editorStore.applicationStore.logService.error( - LogEvent.create(LEGEND_QUERY_APP_EVENT.GENERIC_FAILURE), - error, - ); - editorStore.applicationStore.notificationService.notifyError( - error, - ); - } - }; + }; - editorStore.applicationStore.alertService.setActionAlertInfo({ - message: `To change the data space associated with this query, you need to ${ - query.isCurrentUserQuery - ? 'update the query' - : 'create a new query' - } to proceed`, - type: ActionAlertType.CAUTION, - actions: [ - { - label: query.isCurrentUserQuery - ? 'Update query' - : 'Create new query', - type: ActionAlertActionType.PROCEED_WITH_CAUTION, - handler: () => { - createQuery().catch( - editorStore.applicationStore.alertUnhandledError, - ); + editorStore.applicationStore.alertService.setActionAlertInfo({ + message: `To change the data space associated with this query, you need to ${ + query.isCurrentUserQuery + ? 'update the query' + : 'create a new query' + } to proceed`, + type: ActionAlertType.CAUTION, + actions: [ + { + label: query.isCurrentUserQuery + ? 'Update query' + : 'Create new query', + type: ActionAlertActionType.PROCEED_WITH_CAUTION, + handler: () => { + createQuery().catch( + editorStore.applicationStore.alertUnhandledError, + ); + }, }, - }, - { - label: 'Abort', - type: ActionAlertActionType.PROCEED, - default: true, - }, - ], - }); - } else { - editorStore.applicationStore.notificationService.notifyWarning( - `Can't switch data space: default execution context not specified`, - ); - } - }, - true, - dataSpaceAnalysisResult, - undefined, - undefined, - undefined, - projectInfo, - editorStore.applicationStore.config.options.queryBuilderConfig, + { + label: 'Abort', + type: ActionAlertActionType.PROCEED, + default: true, + }, + ], + }); + } else { + editorStore.applicationStore.notificationService.notifyWarning( + `Can't switch data space: default execution context not specified`, + ); + } + }, + true, + dataSpaceAnalysisResult, + undefined, + undefined, + undefined, + projectInfo, + ); + const mappingModelCoverageAnalysisResult = + dataSpaceAnalysisResult?.executionContextsIndex.get( + matchingExecutionContext.name, + )?.mappingModelCoverageAnalysisResult; + if (mappingModelCoverageAnalysisResult) { + dataSpaceQueryBuilderState.explorerState.mappingModelCoverageAnalysisResult = + mappingModelCoverageAnalysisResult; + } + return dataSpaceQueryBuilderState; + } + } + return undefined; + }, + ]; + } + + override getExtraQueryGraphBuilderGetters(): QueryGraphBuilderGetter[] { + return [ + ( + editorStore: QueryEditorStore, + ): ((editorStore: QueryEditorStore) => GeneratorFn) | undefined => { + if (editorStore instanceof ExistingQueryEditorStore) { + const query = editorStore.query; + const dataSpaceTaggedValue = query?.taggedValues?.find( + (taggedValue) => + taggedValue.profile === QUERY_PROFILE_PATH && + taggedValue.tag === QUERY_PROFILE_TAG_DATA_SPACE && + isValidFullPath(taggedValue.value), ); - const mappingModelCoverageAnalysisResult = - dataSpaceAnalysisResult?.executionContextsIndex.get( - matchingExecutionContext.name, - )?.mappingModelCoverageAnalysisResult; - if (mappingModelCoverageAnalysisResult) { - dataSpaceQueryBuilderState.explorerState.mappingModelCoverageAnalysisResult = - mappingModelCoverageAnalysisResult; + if (dataSpaceTaggedValue) { + return buildGraph; } - return dataSpaceQueryBuilderState; } return undefined; }, diff --git a/packages/legend-extension-dsl-data-space/src/components/query/DataSpaceQueryBuilder.tsx b/packages/legend-extension-dsl-data-space/src/components/query/DataSpaceQueryBuilder.tsx index 4a1f702b2d..dd5ac9b5a4 100644 --- a/packages/legend-extension-dsl-data-space/src/components/query/DataSpaceQueryBuilder.tsx +++ b/packages/legend-extension-dsl-data-space/src/components/query/DataSpaceQueryBuilder.tsx @@ -44,11 +44,16 @@ import { getMappingCompatibleRuntimes, PackageableElementExplicitReference, RuntimePointer, + type PackageableRuntime, } from '@finos/legend-graph'; import type { DataSpaceInfo } from '../../stores/query/DataSpaceInfo.js'; import { generateGAVCoordinates } from '@finos/legend-storage'; import { useEffect, useMemo, useState } from 'react'; -import { debounce, guaranteeType } from '@finos/legend-shared'; +import { + debounce, + guaranteeType, + guaranteeNonNullable, +} from '@finos/legend-shared'; import { flowResult } from 'mobx'; import { DataSpace, @@ -57,6 +62,7 @@ import { import { DataSpaceIcon } from '../DSL_DataSpace_Icon.js'; import { DataSpaceAdvancedSearchModal } from './DataSpaceAdvancedSearchModal.js'; import type { EditorStore } from '@finos/legend-application-studio'; +import { useQueryEditorStore } from '@finos/legend-application-query'; export type DataSpaceOption = { label: string; @@ -105,6 +111,25 @@ export const formatDataSpaceOptionLabel = ( ); +const resolveExecutionContextRuntimes = ( + queryBuilderState: DataSpaceQueryBuilderState, +): PackageableRuntime[] => { + if (queryBuilderState.dataSpaceAnalysisResult) { + const executionContext = Array.from( + queryBuilderState.dataSpaceAnalysisResult.executionContextsIndex.values(), + ).find( + (e) => + e.mapping.path === + queryBuilderState.executionContext.mapping.value.path, + ); + return guaranteeNonNullable(executionContext).compatibleRuntimes; + } + return getMappingCompatibleRuntimes( + queryBuilderState.executionContext.mapping.value, + queryBuilderState.graphManagerState.usableRuntimes, + ); +}; + type ExecutionContextOption = { label: string; value: DataSpaceExecutionContext; @@ -129,6 +154,7 @@ const DataSpaceQueryBuilderSetupPanelContent = observer( (props: { queryBuilderState: DataSpaceQueryBuilderState }) => { const { queryBuilderState } = props; const applicationStore = useApplicationStore(); + const editorStore = useQueryEditorStore(); const [dataSpaceSearchText, setDataSpaceSearchText] = useState(''); // data space @@ -148,8 +174,10 @@ const DataSpaceQueryBuilderSetupPanelContent = observer( queryBuilderState.dataSpace.defaultExecutionContext.name, }, }; - const onDataSpaceOptionChange = (option: DataSpaceOption): void => { - queryBuilderState.onDataSpaceChange(option.value); + const onDataSpaceOptionChange = async ( + option: DataSpaceOption, + ): Promise => { + await queryBuilderState.onDataSpaceChange(option.value); }; // data space search text @@ -180,22 +208,22 @@ const DataSpaceQueryBuilderSetupPanelContent = observer( const selectedExecutionContextOption = buildExecutionContextOption( queryBuilderState.executionContext, ); - const onExecutionContextOptionChange = ( + const onExecutionContextOptionChange = async ( option: ExecutionContextOption, - ): void => { + ): Promise => { if (option.value === queryBuilderState.executionContext) { return; } queryBuilderState.setExecutionContext(option.value); - queryBuilderState.propagateExecutionContextChange(option.value); + await queryBuilderState.propagateExecutionContextChange( + option.value, + editorStore, + ); queryBuilderState.onExecutionContextChange?.(option.value); }; // runtime - const runtimeOptions = getMappingCompatibleRuntimes( - queryBuilderState.executionContext.mapping.value, - queryBuilderState.graphManagerState.usableRuntimes, - ) + const runtimeOptions = resolveExecutionContextRuntimes(queryBuilderState) .map( (rt) => new RuntimePointer(PackageableElementExplicitReference.create(rt)), @@ -225,11 +253,7 @@ const DataSpaceQueryBuilderSetupPanelContent = observer( }); // class - const classes = resolveUsableDataSpaceClasses( - queryBuilderState.dataSpace, - queryBuilderState.executionContext.mapping.value, - queryBuilderState.graphManagerState, - ); + const classes = resolveUsableDataSpaceClasses(queryBuilderState); useEffect(() => { flowResult(queryBuilderState.loadDataSpaces('')).catch( @@ -400,14 +424,15 @@ export const queryDataSpace = async ( const embeddedQueryBuilderState = editorStore.embeddedQueryBuilderState; await flowResult( embeddedQueryBuilderState.setEmbeddedQueryBuilderConfiguration({ - setupQueryBuilderState: () => { + setupQueryBuilderState: async () => { const queryBuilderState = new DataSpaceQueryBuilderState( editorStore.applicationStore, editorStore.graphManagerState, editorStore.depotServerClient, dataSpace, dataSpace.defaultExecutionContext, - (dataSpaceInfo: DataSpaceInfo) => { + false, + async (dataSpaceInfo: DataSpaceInfo) => { queryBuilderState.dataSpace = guaranteeType( queryBuilderState.graphManagerState.graph.getElement( dataSpaceInfo.path, @@ -417,7 +442,7 @@ export const queryDataSpace = async ( queryBuilderState.setExecutionContext( queryBuilderState.dataSpace.defaultExecutionContext, ); - queryBuilderState.propagateExecutionContextChange( + await queryBuilderState.propagateExecutionContextChange( queryBuilderState.dataSpace.defaultExecutionContext, ); }, @@ -432,7 +457,7 @@ export const queryDataSpace = async ( queryBuilderState.setExecutionContext( dataSpace.defaultExecutionContext, ); - queryBuilderState.propagateExecutionContextChange( + await queryBuilderState.propagateExecutionContextChange( dataSpace.defaultExecutionContext, ); return queryBuilderState; diff --git a/packages/legend-extension-dsl-data-space/src/graph-manager/protocol/pure/DSL_DataSpace_PureGraphManagerExtension.ts b/packages/legend-extension-dsl-data-space/src/graph-manager/protocol/pure/DSL_DataSpace_PureGraphManagerExtension.ts index 48a9d17992..bdb0538dd7 100644 --- a/packages/legend-extension-dsl-data-space/src/graph-manager/protocol/pure/DSL_DataSpace_PureGraphManagerExtension.ts +++ b/packages/legend-extension-dsl-data-space/src/graph-manager/protocol/pure/DSL_DataSpace_PureGraphManagerExtension.ts @@ -17,14 +17,17 @@ import { type AbstractPureGraphManager, AbstractPureGraphManagerExtension, + type PureModel, + type GraphManagerOperationReport, } from '@finos/legend-graph'; -import type { Entity } from '@finos/legend-storage'; +import type { Entity, ProjectGAVCoordinates } from '@finos/legend-storage'; import { guaranteeNonNullable, type ActionState, type PlainObject, } from '@finos/legend-shared'; import type { DataSpaceAnalysisResult } from '../../action/analytics/DataSpaceAnalysis.js'; +import type { NotificationService } from '@finos/legend-application'; export abstract class DSL_DataSpace_PureGraphManagerExtension extends AbstractPureGraphManagerExtension { abstract analyzeDataSpace( @@ -34,10 +37,18 @@ export abstract class DSL_DataSpace_PureGraphManagerExtension extends AbstractPu actionState?: ActionState, ): Promise; - abstract retrieveDataSpaceAnalysisFromCache( - cacheRetriever: () => Promise>, + abstract analyzeDataSpaceCoverage( + dataSpacePath: string, + entitiesRetriever: () => Promise, + cacheRetriever?: () => Promise>, actionState?: ActionState, - ): Promise; + graphReport?: GraphManagerOperationReport | undefined, + pureGraph?: PureModel | undefined, + executionContext?: string | undefined, + mappingPath?: string | undefined, + projectInfo?: ProjectGAVCoordinates, + notificationService?: NotificationService | undefined, + ): Promise; } export const DSL_DataSpace_getGraphManagerExtension = ( diff --git a/packages/legend-extension-dsl-data-space/src/graph-manager/protocol/pure/DSL_DataSpace_PureProtocolProcessorPlugin.ts b/packages/legend-extension-dsl-data-space/src/graph-manager/protocol/pure/DSL_DataSpace_PureProtocolProcessorPlugin.ts index ba05ff2db9..e380cba695 100644 --- a/packages/legend-extension-dsl-data-space/src/graph-manager/protocol/pure/DSL_DataSpace_PureProtocolProcessorPlugin.ts +++ b/packages/legend-extension-dsl-data-space/src/graph-manager/protocol/pure/DSL_DataSpace_PureProtocolProcessorPlugin.ts @@ -144,6 +144,29 @@ export class DSL_DataSpace_PureProtocolProcessorPlugin ); return execContext; }); + if (elementProtocol.elements) { + element.elements = elementProtocol.elements.map((pointer) => { + const elementReference = context.resolveElement( + pointer.path, + true, + ); + if ( + elementReference.value instanceof Package || + elementReference.value instanceof Class || + elementReference.value instanceof Enumeration || + elementReference.value instanceof Association + ) { + const elementPointer = new DataSpaceElementPointer(); + elementPointer.element = + elementReference as unknown as PackageableElementReference; + elementPointer.exclude = pointer.exclude; + return elementPointer; + } + throw new UnsupportedOperationError( + `Can't find data space element (only allow packages, classes, enumerations, and associations) '${pointer.path}'`, + ); + }); + } element.defaultExecutionContext = guaranteeNonNullable( element.executionContexts.find( (execContext) => @@ -180,29 +203,6 @@ export class DSL_DataSpace_PureProtocolProcessorPlugin .filter(isNonNullable); element.title = elementProtocol.title; element.description = elementProtocol.description; - if (elementProtocol.elements) { - element.elements = elementProtocol.elements.map((pointer) => { - const elementReference = context.resolveElement( - pointer.path, - true, - ); - if ( - elementReference.value instanceof Package || - elementReference.value instanceof Class || - elementReference.value instanceof Enumeration || - elementReference.value instanceof Association - ) { - const elementPointer = new DataSpaceElementPointer(); - elementPointer.element = - elementReference as unknown as PackageableElementReference; - elementPointer.exclude = pointer.exclude; - return elementPointer; - } - throw new UnsupportedOperationError( - `Can't find data space element (only allow packages, classes, enumerations, and associations) '${pointer.path}'`, - ); - }); - } if (elementProtocol.executables) { element.executables = elementProtocol.executables.map( (executableProtocol) => { diff --git a/packages/legend-extension-dsl-data-space/src/graph-manager/protocol/pure/v1/V1_DSL_DataSpace_PureGraphManagerExtension.ts b/packages/legend-extension-dsl-data-space/src/graph-manager/protocol/pure/v1/V1_DSL_DataSpace_PureGraphManagerExtension.ts index 03b3a2f59b..f6821942ef 100644 --- a/packages/legend-extension-dsl-data-space/src/graph-manager/protocol/pure/v1/V1_DSL_DataSpace_PureGraphManagerExtension.ts +++ b/packages/legend-extension-dsl-data-space/src/graph-manager/protocol/pure/v1/V1_DSL_DataSpace_PureGraphManagerExtension.ts @@ -31,8 +31,10 @@ import { V1_buildDatasetSpecification, type PureProtocolProcessorPlugin, V1_buildModelCoverageAnalysisResult, + type GraphManagerOperationReport, + LegendSDLC, } from '@finos/legend-graph'; -import type { Entity } from '@finos/legend-storage'; +import type { Entity, ProjectGAVCoordinates } from '@finos/legend-storage'; import { ActionState, assertErrorThrown, @@ -83,6 +85,8 @@ import { V1_DataSpaceMultiExecutionServiceExecutableInfo, } from './engine/analytics/V1_DataSpaceAnalysis.js'; import { getDiagram } from '@finos/legend-extension-dsl-diagram/graph'; +import { resolveVersion } from '@finos/legend-server-depot'; +import type { NotificationService } from '@finos/legend-application'; const ANALYZE_DATA_SPACE_TRACE = 'analyze data space'; const TEMPORARY__TDS_SAMPLE_VALUES__DELIMETER = '-- e.g.'; @@ -141,20 +145,68 @@ export class V1_DSL_DataSpace_PureGraphManagerExtension extends DSL_DataSpace_Pu ); } - async retrieveDataSpaceAnalysisFromCache( - cacheRetriever: () => Promise>, + async analyzeDataSpaceCoverage( + dataSpacePath: string, + entitiesRetriever: () => Promise, + cacheRetriever?: () => Promise>, actionState?: ActionState, - ): Promise { - const cacheResult = await this.fetchDataSpaceAnalysisFromCache( - cacheRetriever, - actionState, - ); - return cacheResult - ? this.buildDataSpaceAnalytics( - cacheResult, - this.graphManager.pluginManager.getPureProtocolProcessorPlugins(), - ) + graphReport?: GraphManagerOperationReport | undefined, + pureGraph?: PureModel | undefined, + executionContext?: string | undefined, + mappingPath?: string | undefined, + projectInfo?: ProjectGAVCoordinates, + notificationService?: NotificationService | undefined, + ): Promise { + const cacheResult = cacheRetriever + ? await this.fetchDataSpaceAnalysisFromCache(cacheRetriever, actionState) : undefined; + const engineClient = this.graphManager.engine.getEngineServerClient(); + notificationService?.notify( + `Please release a new version of the project and create a new query from that to reduce the load time`, + ); + let analysisResult: PlainObject; + if ( + cacheResult && + V1_deserializeDataSpaceAnalysisResult( + cacheResult, + this.graphManager.pluginManager.getPureProtocolProcessorPlugins(), + ).executionContexts.every( + (e) => e.mappingModelCoverageAnalysisResult.model !== undefined, + ) + ) { + analysisResult = cacheResult; + } else { + actionState?.setMessage('Fetching project entities and dependencies...'); + const entities = await entitiesRetriever(); + actionState?.setMessage('Analyzing data space...'); + analysisResult = await engineClient.postWithTracing< + PlainObject + >( + engineClient.getTraceData(ANALYZE_DATA_SPACE_TRACE), + `${engineClient._pure()}/analytics/dataSpace/coverage`, + { + clientVersion: V1_PureGraphManager.DEV_PROTOCOL_VERSION, + dataSpace: dataSpacePath, + model: { + _type: V1_PureModelContextType.DATA, + elements: entities.map((entity) => entity.content), + }, + }, + {}, + undefined, + undefined, + { enableCompression: true }, + ); + } + return this.buildDataSpaceAnalytics( + analysisResult, + this.graphManager.pluginManager.getPureProtocolProcessorPlugins(), + graphReport, + pureGraph, + executionContext, + mappingPath, + projectInfo, + ); } private async fetchDataSpaceAnalysisFromCache( @@ -177,11 +229,19 @@ export class V1_DSL_DataSpace_PureGraphManagerExtension extends DSL_DataSpace_Pu return cacheResult; } - private async buildDataSpaceAnalytics( - json: PlainObject, + async buildDataSpaceAnalytics( + analytics: PlainObject, plugins: PureProtocolProcessorPlugin[], + graphReport?: GraphManagerOperationReport | undefined, + pureGraph?: PureModel | undefined, + executionContext?: string | undefined, + mappingPath?: string | undefined, + projectInfo?: ProjectGAVCoordinates, ): Promise { - const analysisResult = V1_deserializeDataSpaceAnalysisResult(json, plugins); + const analysisResult = V1_deserializeDataSpaceAnalysisResult( + analytics, + plugins, + ); const result = new DataSpaceAnalysisResult(); result.name = analysisResult.name; result.package = analysisResult.package; @@ -228,25 +288,32 @@ export class V1_DSL_DataSpace_PureGraphManagerExtension extends DSL_DataSpace_Pu // NOTE: we will relax the check and not throw here for unknown support info type } - // create an empty graph - const extensionElementClasses = this.graphManager.pluginManager - .getPureGraphPlugins() - .flatMap((plugin) => plugin.getExtraPureGraphExtensionClasses?.() ?? []); - const systemModel = new SystemModel(extensionElementClasses); - const coreModel = new CoreModel(extensionElementClasses); - await this.graphManager.buildSystem( - coreModel, - systemModel, - ActionState.create(), - {}, - ); - systemModel.initializeAutoImports(); - const graph = new PureModel( - coreModel, - systemModel, - this.graphManager.pluginManager.getPureGraphPlugins(), - ); - + let graphEntities; + let graph: PureModel; + if (pureGraph) { + graph = pureGraph; + } else { + // create an empty graph + const extensionElementClasses = this.graphManager.pluginManager + .getPureGraphPlugins() + .flatMap( + (plugin) => plugin.getExtraPureGraphExtensionClasses?.() ?? [], + ); + const systemModel = new SystemModel(extensionElementClasses); + const coreModel = new CoreModel(extensionElementClasses); + await this.graphManager.buildSystem( + coreModel, + systemModel, + ActionState.create(), + {}, + ); + systemModel.initializeAutoImports(); + graph = new PureModel( + coreModel, + systemModel, + this.graphManager.pluginManager.getPureGraphPlugins(), + ); + } // Create dummy mappings and runtimes // TODO?: these stubbed mappings and runtimes are not really useful that useful, so either we should // simplify the model here or potentially refactor the backend analytics endpoint to return these as model @@ -275,56 +342,86 @@ export class V1_DSL_DataSpace_PureGraphManagerExtension extends DSL_DataSpace_Pu runtime.runtimeValue = new V1_EngineRuntime(); return runtime; }); - - // prepare the model context data - const graphEntities = analysisResult.model.elements - // NOTE: this is a temporary hack to fix a problem with data space analytics - // where the classes for properties are not properly surveyed - // We need to wait for the actual fix in backend to be merged and released - // See https://github.com/finos/legend-engine/pull/836 - .concat( - uniq( - analysisResult.model.elements.flatMap((element) => { - if (element instanceof V1_Class) { - return element.derivedProperties - .map((prop) => prop.returnType) - .concat(element.properties.map((prop) => prop.type)); - } - return []; - }), - ) - // make sure to not include types already returned by the analysis - .filter( - (path) => - !analysisResult.model.elements - .map((el) => el.path) - .includes(path), + const mappingModelCoverageAnalysisResult = + analysisResult.executionContexts.find( + (value) => value.name === executionContext, + )?.mappingModelCoverageAnalysisResult ?? + analysisResult.executionContexts.find( + (value) => value.mapping === mappingPath, + )?.mappingModelCoverageAnalysisResult; + const pmcd = mappingModelCoverageAnalysisResult?.model; + if (pmcd && projectInfo) { + graphEntities = pmcd.elements + .concat(mappingModels) + .concat(runtimeModels) + // NOTE: if an element could be found in the graph already it means it comes from system + // so we could rid of it + .filter((el) => !graph.getNullableElement(el.path, false)) + .map((el) => this.graphManager.elementProtocolToEntity(el)); + await this.graphManager.buildGraphForQuery( + graph, + graphEntities, + ActionState.create(), + { + origin: new LegendSDLC( + projectInfo.groupId, + projectInfo.artifactId, + resolveVersion(projectInfo.versionId), + ), + }, + graphReport, + ); + } else { + // prepare the model context data + graphEntities = analysisResult.model.elements + // NOTE: this is a temporary hack to fix a problem with data space analytics + // where the classes for properties are not properly surveyed + // We need to wait for the actual fix in backend to be merged and released + // See https://github.com/finos/legend-engine/pull/836 + .concat( + uniq( + analysisResult.model.elements.flatMap((element) => { + if (element instanceof V1_Class) { + return element.derivedProperties + .map((prop) => prop.returnType) + .concat(element.properties.map((prop) => prop.type)); + } + return []; + }), ) - .map((path) => { - const [pkgPath, name] = resolvePackagePathAndElementName(path); - if (!pkgPath) { - // exclude package-less elements (i.e. primitive types) - return undefined; - } - const _class = new V1_Class(); - _class.name = name; - _class.package = pkgPath; - return _class; - }) - .filter(isNonNullable), - ) - .concat(mappingModels) - .concat(runtimeModels) - // NOTE: if an element could be found in the graph already it means it comes from system - // so we could rid of it - .filter((el) => !graph.getNullableElement(el.path, false)) - .map((el) => this.graphManager.elementProtocolToEntity(el)); + // make sure to not include types already returned by the analysis + .filter( + (path) => + !analysisResult.model.elements + .map((el) => el.path) + .includes(path), + ) + .map((path) => { + const [pkgPath, name] = resolvePackagePathAndElementName(path); + if (!pkgPath) { + // exclude package-less elements (i.e. primitive types) + return undefined; + } + const _class = new V1_Class(); + _class.name = name; + _class.package = pkgPath; + return _class; + }) + .filter(isNonNullable), + ) + .concat(mappingModels) + .concat(runtimeModels) + // NOTE: if an element could be found in the graph already it means it comes from system + // so we could rid of it + .filter((el) => !graph.getNullableElement(el.path, false)) + .map((el) => this.graphManager.elementProtocolToEntity(el)); - await this.graphManager.buildGraph( - graph, - graphEntities, - ActionState.create(), - ); + await this.graphManager.buildGraph( + graph, + graphEntities, + ActionState.create(), + ); + } // execution context result.executionContextsIndex = new Map< @@ -344,6 +441,7 @@ export class V1_DSL_DataSpace_PureGraphManagerExtension extends DSL_DataSpace_Pu contextAnalysisResult.mappingModelCoverageAnalysisResult = V1_buildModelCoverageAnalysisResult( context.mappingModelCoverageAnalysisResult, + this.graphManager, contextAnalysisResult.mapping, ); contextAnalysisResult.compatibleRuntimes = context.compatibleRuntimes.map( diff --git a/packages/legend-extension-dsl-data-space/src/stores/query/DataSpaceQueryBuilderState.ts b/packages/legend-extension-dsl-data-space/src/stores/query/DataSpaceQueryBuilderState.ts index 6a1a189e1a..55ebb2c196 100644 --- a/packages/legend-extension-dsl-data-space/src/stores/query/DataSpaceQueryBuilderState.ts +++ b/packages/legend-extension-dsl-data-space/src/stores/query/DataSpaceQueryBuilderState.ts @@ -29,12 +29,20 @@ import { RuntimePointer, type Runtime, Class, - type Mapping, getDescendantsOfPackage, Package, + createGraphBuilderReport, + GRAPH_MANAGER_EVENT, + LegendSDLC, + V1_EngineRuntime, + V1_Mapping, + V1_PackageableRuntime, + V1_PureGraphManager, + resolvePackagePathAndElementName, } from '@finos/legend-graph'; import { DepotScope, + resolveVersion, SNAPSHOT_VERSION_ALIAS, type DepotServerClient, type StoredEntity, @@ -46,8 +54,12 @@ import { getNullableFirstEntry, filterByType, uniq, + StopWatch, + LogEvent, + guaranteeNonNullable, + guaranteeType, } from '@finos/legend-shared'; -import { action, flow, makeObservable, observable } from 'mobx'; +import { action, flow, flowResult, makeObservable, observable } from 'mobx'; import { renderDataSpaceQueryBuilderSetupPanelContent } from '../../components/query/DataSpaceQueryBuilder.js'; import { DataSpace, @@ -57,12 +69,17 @@ import { DATA_SPACE_ELEMENT_CLASSIFIER_PATH } from '../../graph-manager/protocol import { type DataSpaceInfo, extractDataSpaceInfo } from './DataSpaceInfo.js'; import { DataSpaceAdvancedSearchState } from './DataSpaceAdvancedSearchState.js'; import type { DataSpaceAnalysisResult } from '../../graph-manager/action/analytics/DataSpaceAnalysis.js'; +import { + LEGEND_QUERY_APP_EVENT, + type QueryEditorStore, +} from '@finos/legend-application-query'; export const resolveUsableDataSpaceClasses = ( - dataSpace: DataSpace, - mapping: Mapping, - graphManagerState: GraphManagerState, + queryBuilderState: DataSpaceQueryBuilderState, ): Class[] => { + const dataSpace = queryBuilderState.dataSpace; + const mapping = queryBuilderState.executionContext.mapping.value; + const graphManagerState = queryBuilderState.graphManagerState; if (dataSpace.elements?.length) { const dataSpaceElements = dataSpace.elements.map((ep) => ep.element.value); return uniq([ @@ -73,6 +90,24 @@ export const resolveUsableDataSpaceClasses = ( .flat() .filter(filterByType(Class)), ]); + } else if ( + queryBuilderState.explorerState.mappingModelCoverageAnalysisResult && + // This check is to make sure that we have `info` field present in `MappedEntity` which + // contains information about the mapped class path + queryBuilderState.explorerState.mappingModelCoverageAnalysisResult.mappedEntities.some( + (m) => m.info !== undefined, + ) + ) { + const compatibleClassPaths = + queryBuilderState.explorerState.mappingModelCoverageAnalysisResult.mappedEntities.map( + (e) => e.info?.classPath, + ); + const uniqueCompatibleClasses = compatibleClassPaths.filter( + (val, index) => compatibleClassPaths.indexOf(val) === index, + ); + return graphManagerState.graph.classes.filter((c) => + uniqueCompatibleClasses.includes(c.path), + ); } return getMappingCompatibleClasses(mapping, graphManagerState.usableClasses); }; @@ -121,7 +156,7 @@ export class DataSpaceQueryBuilderState extends QueryBuilderState { readonly depotServerClient: DepotServerClient; readonly isAdvancedDataSpaceSearchEnabled: boolean; readonly loadDataSpacesState = ActionState.create(); - readonly onDataSpaceChange: (val: DataSpaceInfo) => void; + readonly onDataSpaceChange: (val: DataSpaceInfo) => Promise; readonly onExecutionContextChange?: | ((val: DataSpaceExecutionContext) => void) | undefined; @@ -138,6 +173,7 @@ export class DataSpaceQueryBuilderState extends QueryBuilderState { dataSpaces: DataSpaceInfo[] = []; showRuntimeSelector = false; advancedSearchState?: DataSpaceAdvancedSearchState | undefined; + isLightGraphEnabled!: boolean; constructor( applicationStore: GenericLegendApplicationStore, @@ -145,7 +181,8 @@ export class DataSpaceQueryBuilderState extends QueryBuilderState { depotServerClient: DepotServerClient, dataSpace: DataSpace, executionContext: DataSpaceExecutionContext, - onDataSpaceChange: (val: DataSpaceInfo) => void, + isLightGraphEnabled: boolean, + onDataSpaceChange: (val: DataSpaceInfo) => Promise, isAdvancedDataSpaceSearchEnabled: boolean, dataSpaceAnalysisResult?: DataSpaceAnalysisResult | undefined, onExecutionContextChange?: @@ -161,11 +198,13 @@ export class DataSpaceQueryBuilderState extends QueryBuilderState { makeObservable(this, { dataSpaces: observable, executionContext: observable, + isLightGraphEnabled: observable, showRuntimeSelector: observable, advancedSearchState: observable, showAdvancedSearchPanel: action, hideAdvancedSearchPanel: action, setExecutionContext: action, + setIsLightGraphEnabled: action, setShowRuntimeSelector: action, loadDataSpaces: flow, }); @@ -174,6 +213,7 @@ export class DataSpaceQueryBuilderState extends QueryBuilderState { this.dataSpace = dataSpace; this.executionContext = executionContext; this.projectInfo = projectInfo; + this.isLightGraphEnabled = isLightGraphEnabled; this.onDataSpaceChange = onDataSpaceChange; this.onExecutionContextChange = onExecutionContextChange; this.onRuntimeChange = onRuntimeChange; @@ -188,6 +228,10 @@ export class DataSpaceQueryBuilderState extends QueryBuilderState { : 'query-builder__setup__data-space'; } + setIsLightGraphEnabled(val: boolean): void { + this.isLightGraphEnabled = val; + } + showAdvancedSearchPanel(): void { if (this.projectInfo && this.isAdvancedDataSpaceSearchEnabled) { this.advancedSearchState = new DataSpaceAdvancedSearchState( @@ -275,26 +319,134 @@ export class DataSpaceQueryBuilderState extends QueryBuilderState { * - If no class is chosen, try to choose a compatible class * - If the chosen class is compatible with the new selected execution context mapping, do nothing, otherwise, try to choose a compatible class */ - propagateExecutionContextChange( + async propagateExecutionContextChange( executionContext: DataSpaceExecutionContext, - ): void { + editorStore?: QueryEditorStore, + isGraphBuildingNotRequired?: boolean, + ): Promise { const mapping = executionContext.mapping.value; - this.changeMapping(mapping); const mappingModelCoverageAnalysisResult = this.dataSpaceAnalysisResult?.executionContextsIndex.get( executionContext.name, )?.mappingModelCoverageAnalysisResult; - if (mappingModelCoverageAnalysisResult) { + if (this.dataSpaceAnalysisResult && mappingModelCoverageAnalysisResult) { + if (!isGraphBuildingNotRequired && editorStore) { + try { + const stopWatch = new StopWatch(); + const graph = this.graphManagerState.createNewGraph(); + + const graph_buildReport = createGraphBuilderReport(); + // Create dummy mappings and runtimes + // TODO?: these stubbed mappings and runtimes are not really useful that useful, so either we should + // simplify the model here or potentially refactor the backend analytics endpoint to return these as model + const mappingModels = uniq( + Array.from( + this.dataSpaceAnalysisResult.executionContextsIndex.values(), + ).map((context) => context.mapping), + ).map((m) => { + const _mapping = new V1_Mapping(); + const [packagePath, name] = resolvePackagePathAndElementName( + m.path, + ); + _mapping.package = packagePath; + _mapping.name = name; + return guaranteeType( + this.graphManagerState.graphManager, + V1_PureGraphManager, + ).elementProtocolToEntity(_mapping); + }); + const runtimeModels = uniq( + Array.from( + this.dataSpaceAnalysisResult.executionContextsIndex.values(), + ) + .map((context) => context.defaultRuntime) + .concat( + Array.from( + this.dataSpaceAnalysisResult.executionContextsIndex.values(), + ).flatMap((val) => val.compatibleRuntimes), + ), + ).map((r) => { + const runtime = new V1_PackageableRuntime(); + const [packagePath, name] = resolvePackagePathAndElementName( + r.path, + ); + runtime.package = packagePath; + runtime.name = name; + runtime.runtimeValue = new V1_EngineRuntime(); + return guaranteeType( + this.graphManagerState.graphManager, + V1_PureGraphManager, + ).elementProtocolToEntity(runtime); + }); + const graphEntities = guaranteeNonNullable( + mappingModelCoverageAnalysisResult.entities, + ) + .concat(mappingModels) + .concat(runtimeModels) + // NOTE: if an element could be found in the graph already it means it comes from system + // so we could rid of it + .filter( + (el) => + !graph.getNullableElement(el.path, false) && + !el.path.startsWith('meta::'), + ); + await this.graphManagerState.graphManager.buildGraphForQuery( + graph, + graphEntities, + ActionState.create(), + { + origin: new LegendSDLC( + guaranteeNonNullable(this.projectInfo).groupId, + guaranteeNonNullable(this.projectInfo).artifactId, + resolveVersion( + guaranteeNonNullable(this.projectInfo).versionId, + ), + ), + }, + graph_buildReport, + ); + this.graphManagerState.graph = graph; + const dependency_buildReport = createGraphBuilderReport(); + // report + stopWatch.record(GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH__SUCCESS); + const graphBuilderReportData = { + timings: + this.applicationStore.timeService.finalizeTimingsRecord( + stopWatch, + ), + dependencies: dependency_buildReport, + dependenciesCount: + this.graphManagerState.graph.dependencyManager + .numberOfDependencies, + graph: graph_buildReport, + }; + editorStore.logBuildGraphMetrics(graphBuilderReportData); + + this.applicationStore.logService.info( + LogEvent.create(GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH__SUCCESS), + graphBuilderReportData, + ); + } catch (error) { + assertErrorThrown(error); + editorStore.applicationStore.logService.error( + LogEvent.create(LEGEND_QUERY_APP_EVENT.GENERIC_FAILURE), + error, + ); + + editorStore.graphManagerState.graph = + editorStore.graphManagerState.createNewGraph(); + await flowResult(editorStore.buildFullGraph()); + } + } this.explorerState.mappingModelCoverageAnalysisResult = mappingModelCoverageAnalysisResult; } + const compatibleClasses = resolveUsableDataSpaceClasses(this); + + this.changeMapping(mapping); + this.changeRuntime(new RuntimePointer(executionContext.defaultRuntime)); - const compatibleClasses = resolveUsableDataSpaceClasses( - this.dataSpace, - mapping, - this.graphManagerState, - ); // if there is no chosen class or the chosen one is not compatible // with the mapping then pick a compatible class if possible if (!this.class || !compatibleClasses.includes(this.class)) { @@ -303,5 +455,6 @@ export class DataSpaceQueryBuilderState extends QueryBuilderState { this.changeClass(possibleNewClass); } } + this.explorerState.refreshTreeData(); } } diff --git a/packages/legend-extension-dsl-data-space/src/stores/query/DataSpaceQueryCreatorStore.ts b/packages/legend-extension-dsl-data-space/src/stores/query/DataSpaceQueryCreatorStore.ts index 579d752d36..d0fb4a4c71 100644 --- a/packages/legend-extension-dsl-data-space/src/stores/query/DataSpaceQueryCreatorStore.ts +++ b/packages/legend-extension-dsl-data-space/src/stores/query/DataSpaceQueryCreatorStore.ts @@ -23,6 +23,8 @@ import { type Runtime, type Class, type RawLambda, + createGraphBuilderReport, + GRAPH_MANAGER_EVENT, } from '@finos/legend-graph'; import { QueryEditorStore, @@ -30,15 +32,20 @@ import { type LegendQueryApplicationStore, createViewProjectHandler, createViewSDLCProjectHandler, + LEGEND_QUERY_APP_EVENT, } from '@finos/legend-application-query'; import { type DepotServerClient, StoreProjectData, + retrieveProjectEntitiesWithDependencies, } from '@finos/legend-server-depot'; import { guaranteeNonNullable, guaranteeType, + LogEvent, + StopWatch, uuid, + type GeneratorFn, } from '@finos/legend-shared'; import { QUERY_PROFILE_PATH, @@ -57,6 +64,7 @@ import type { QueryBuilderState } from '@finos/legend-query-builder'; import type { ProjectGAVCoordinates } from '@finos/legend-storage'; import { DSL_DataSpace_getGraphManagerExtension } from '../../graph-manager/protocol/pure/DSL_DataSpace_PureGraphManagerExtension.js'; import { retrieveAnalyticsResultCache } from '../../graph-manager/action/analytics/DataSpaceAnalysisHelper.js'; +import { flowResult } from 'mobx'; export const createQueryDataSpaceTaggedValue = ( dataSpacePath: string, @@ -117,7 +125,83 @@ export class DataSpaceQueryCreatorStore extends QueryEditorStore { }; } + override requiresGraphBuilding(): boolean { + return false; + } + async initializeQueryBuilderState(): Promise { + let dataSpaceAnalysisResult; + let isLightGraphEnabled = true; + try { + const stopWatch = new StopWatch(); + const project = StoreProjectData.serialization.fromJson( + await this.depotServerClient.getProject(this.groupId, this.artifactId), + ); + this.initState.setMessage('Fetching dataspace analysis result'); + // initialize system + stopWatch.record(); + await this.graphManagerState.initializeSystem(); + stopWatch.record(GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH_SYSTEM__SUCCESS); + + const graph_buildReport = createGraphBuilderReport(); + const dependency_buildReport = createGraphBuilderReport(); + dataSpaceAnalysisResult = await DSL_DataSpace_getGraphManagerExtension( + this.graphManagerState.graphManager, + ).analyzeDataSpaceCoverage( + this.dataSpacePath, + () => + retrieveProjectEntitiesWithDependencies( + project, + this.versionId, + this.depotServerClient, + ), + () => + retrieveAnalyticsResultCache( + project, + this.versionId, + this.dataSpacePath, + this.depotServerClient, + ), + undefined, + graph_buildReport, + this.graphManagerState.graph, + this.executionContext, + undefined, + this.getProjectInfo(), + this.applicationStore.notificationService, + ); + const mappingPath = dataSpaceAnalysisResult.executionContextsIndex.get( + this.executionContext, + )?.mapping; + if (mappingPath) { + // report + stopWatch.record(GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH__SUCCESS); + const graphBuilderReportData = { + timings: + this.applicationStore.timeService.finalizeTimingsRecord(stopWatch), + dependencies: dependency_buildReport, + dependenciesCount: + this.graphManagerState.graph.dependencyManager.numberOfDependencies, + graph: graph_buildReport, + }; + this.applicationStore.logService.info( + LogEvent.create(GRAPH_MANAGER_EVENT.INITIALIZE_GRAPH__SUCCESS), + graphBuilderReportData, + ); + } else { + isLightGraphEnabled = false; + this.graphManagerState.graph = this.graphManagerState.createNewGraph(); + await flowResult(this.buildFullGraph()); + } + } catch (error) { + this.applicationStore.logService.error( + LogEvent.create(LEGEND_QUERY_APP_EVENT.GENERIC_FAILURE), + error, + ); + this.graphManagerState.graph = this.graphManagerState.createNewGraph(); + isLightGraphEnabled = false; + await flowResult(this.buildFullGraph()); + } const dataSpace = getDataSpace( this.dataSpacePath, this.graphManagerState.graph, @@ -128,24 +212,6 @@ export class DataSpaceQueryCreatorStore extends QueryEditorStore { ), `Can't find execution context '${this.executionContext}'`, ); - let dataSpaceAnalysisResult; - try { - const project = StoreProjectData.serialization.fromJson( - await this.depotServerClient.getProject(this.groupId, this.artifactId), - ); - dataSpaceAnalysisResult = await DSL_DataSpace_getGraphManagerExtension( - this.graphManagerState.graphManager, - ).retrieveDataSpaceAnalysisFromCache(() => - retrieveAnalyticsResultCache( - project, - this.versionId, - dataSpace.path, - this.depotServerClient, - ), - ); - } catch { - // do nothing - } const projectInfo = new DataSpaceProjectInfo( this.groupId, this.artifactId, @@ -162,7 +228,8 @@ export class DataSpaceQueryCreatorStore extends QueryEditorStore { this.depotServerClient, dataSpace, executionContext, - (dataSpaceInfo: DataSpaceInfo) => { + isLightGraphEnabled, + async (dataSpaceInfo: DataSpaceInfo) => { if (dataSpaceInfo.defaultExecutionContext) { this.applicationStore.navigationService.navigator.goToLocation( generateDataSpaceQueryCreatorRoute( @@ -245,7 +312,11 @@ export class DataSpaceQueryCreatorStore extends QueryEditorStore { projectInfo, ); queryBuilderState.setExecutionContext(executionContext); - queryBuilderState.propagateExecutionContextChange(executionContext); + await queryBuilderState.propagateExecutionContextChange( + executionContext, + this, + true, + ); // set runtime if already chosen if (this.runtimePath) { @@ -292,4 +363,8 @@ export class DataSpaceQueryCreatorStore extends QueryEditorStore { }, }; } + + override *buildGraph(): GeneratorFn { + // do nothing + } } diff --git a/packages/legend-graph/src/graph-manager/AbstractPureGraphManager.ts b/packages/legend-graph/src/graph-manager/AbstractPureGraphManager.ts index 1ac8a1956f..6d00ecf13e 100644 --- a/packages/legend-graph/src/graph-manager/AbstractPureGraphManager.ts +++ b/packages/legend-graph/src/graph-manager/AbstractPureGraphManager.ts @@ -254,6 +254,17 @@ export abstract class AbstractPureGraphManager { report?: GraphManagerOperationReport, ): Promise; + /** + * Process entities and build the main graph for query. + */ + abstract buildGraphForQuery( + graph: PureModel, + entities: Entity[], + buildState: ActionState, + options?: GraphBuilderOptions, + report?: GraphManagerOperationReport, + ): Promise; + /** * Process entities and build the main graph. */ diff --git a/packages/legend-graph/src/graph-manager/action/analytics/MappingModelCoverageAnalysis.ts b/packages/legend-graph/src/graph-manager/action/analytics/MappingModelCoverageAnalysis.ts index 38808ac627..8d8ed66a75 100644 --- a/packages/legend-graph/src/graph-manager/action/analytics/MappingModelCoverageAnalysis.ts +++ b/packages/legend-graph/src/graph-manager/action/analytics/MappingModelCoverageAnalysis.ts @@ -14,19 +14,40 @@ * limitations under the License. */ +import type { Entity } from '@finos/legend-storage'; import type { Mapping } from '../../../graph/metamodel/pure/packageableElements/mapping/Mapping.js'; export type RawMappingModelCoverageAnalysisResult = object; +export class MappedEntityInfo { + readonly __PROPERTIES_INDEX = new Map(); + + classPath: string; + isRootEntity: boolean; + subClasses: string[]; + + constructor(classPath: string, isRootEntity: boolean, subClasses: string[]) { + this.isRootEntity = isRootEntity; + this.subClasses = subClasses; + this.classPath = classPath; + } +} + export class MappedEntity { readonly __PROPERTIES_INDEX = new Map(); path: string; properties: MappedProperty[]; + info?: MappedEntityInfo | undefined; - constructor(path: string, properties: MappedProperty[]) { + constructor( + path: string, + properties: MappedProperty[], + info?: MappedEntityInfo | undefined, + ) { this.path = path; this.properties = properties; + this.info = info; properties.forEach((property) => this.__PROPERTIES_INDEX.set(property.name, property), ); @@ -73,12 +94,18 @@ export class MappingModelCoverageAnalysisResult { readonly mapping: Mapping; mappedEntities: MappedEntity[]; + entities?: Entity[] | undefined; - constructor(mappedEntities: MappedEntity[], mapping: Mapping) { + constructor( + mappedEntities: MappedEntity[], + mapping: Mapping, + entities?: Entity[] | undefined, + ) { this.mappedEntities = mappedEntities; this.mapping = mapping; mappedEntities.forEach((entity) => this.__ENTITIES_INDEX.set(entity.path, entity), ); + this.entities = entities; } } diff --git a/packages/legend-graph/src/graph-manager/action/query/Query.ts b/packages/legend-graph/src/graph-manager/action/query/Query.ts index a168113618..af9ce1c11b 100644 --- a/packages/legend-graph/src/graph-manager/action/query/Query.ts +++ b/packages/legend-graph/src/graph-manager/action/query/Query.ts @@ -14,10 +14,6 @@ * limitations under the License. */ -import type { Mapping } from '../../../graph/metamodel/pure/packageableElements/mapping/Mapping.js'; -import type { PackageableElementReference } from '../../../graph/metamodel/pure/packageableElements/PackageableElementReference.js'; -import type { PackageableRuntime } from '../../../graph/metamodel/pure/packageableElements/runtime/PackageableRuntime.js'; - export class QueryTaggedValue { profile!: string; tag!: string; @@ -40,8 +36,9 @@ export class Query { versionId!: string; groupId!: string; artifactId!: string; - mapping!: PackageableElementReference; - runtime!: PackageableElementReference; + // NOTE: Query can be built before the actual graph is built so we can't have the reference of metamodels here + mapping!: string; + runtime!: string; // We enforce a single owner, for collaboration on query, use Studio // if not owner is specified, any user can own the query // NOTE: the owner is managed automatically by the backend diff --git a/packages/legend-graph/src/graph-manager/protocol/pure/v1/V1_PureGraphManager.ts b/packages/legend-graph/src/graph-manager/protocol/pure/v1/V1_PureGraphManager.ts index 2dafd9d28d..0138b4acaf 100644 --- a/packages/legend-graph/src/graph-manager/protocol/pure/v1/V1_PureGraphManager.ts +++ b/packages/legend-graph/src/graph-manager/protocol/pure/v1/V1_PureGraphManager.ts @@ -797,6 +797,93 @@ export class V1_PureGraphManager extends AbstractPureGraphManager { } } + async buildGraphForQuery( + graph: PureModel, + entities: Entity[], + buildState: ActionState, + options?: GraphBuilderOptions, + _report?: GraphManagerOperationReport, + ): Promise { + const stopWatch = new StopWatch(); + const report = _report ?? createGraphBuilderReport(); + buildState.reset(); + + try { + // deserialize + buildState.setMessage(`Deserializing elements...`); + const data = new V1_PureModelContextData(); + await V1_entitiesToPureModelContextData( + entities, + data, + this.pluginManager.getPureProtocolProcessorPlugins(), + this.subtypeInfo, + ); + stopWatch.record( + GRAPH_MANAGER_EVENT.GRAPH_BUILDER_DESERIALIZE_ELEMENTS__SUCCESS, + ); + + // prepare build inputs + const buildInputs: V1_PureGraphBuilderInput[] = [ + { + model: graph, + data: V1_indexPureModelContextData( + report, + data, + this.graphBuilderExtensions, + ), + }, + ]; + + // index + buildState.setMessage( + `Indexing ${report.elementCount.total} elements...`, + ); + await this.initializeAndIndexElements(graph, buildInputs, options); + stopWatch.record( + GRAPH_MANAGER_EVENT.GRAPH_BUILDER_INDEX_ELEMENTS__SUCCESS, + ); + // build types + buildState.setMessage(`Building domain models...`); + await this.buildTypes(graph, buildInputs, options); + stopWatch.record( + GRAPH_MANAGER_EVENT.GRAPH_BUILDER_BUILD_DOMAIN_MODELS__SUCCESS, + ); + + stopWatch.record( + GRAPH_MANAGER_EVENT.GRAPH_BUILDER_BUILD_OTHER_ELEMENTS__SUCCESS, + ); + + if (options?.origin) { + graph.setOrigin(options.origin); + } + + buildState.pass(); + + const totalTime = stopWatch.elapsed; + report.timings = { + ...Object.fromEntries(stopWatch.records), + [GRAPH_MANAGER_EVENT.GRAPH_BUILDER_BUILD_GRAPH__SUCCESS]: totalTime, + total: totalTime, + }; + } catch (error) { + assertErrorThrown(error); + this.logService.error( + LogEvent.create(GRAPH_MANAGER_EVENT.GRAPH_BUILDER_FAILURE), + error, + ); + buildState.fail(); + /** + * Wrap all error with `GraphBuilderError`, as we throw a lot of assertion error in the graph builder + * But we might want to rethink this decision in the future and throw appropriate type of error + */ + throw error instanceof GraphBuilderError + ? error + : new GraphBuilderError(error); + } finally { + buildState.setMessage(undefined); + } + } + async buildGraph( graph: PureModel, entities: Entity[], @@ -2962,6 +3049,7 @@ export class V1_PureGraphManager extends AbstractPureGraphManager { : this.getFullGraphModelData(graph); return V1_buildModelCoverageAnalysisResult( await this.engine.analyzeMappingModelCoverage(input), + this, mapping, ); } @@ -2975,6 +3063,7 @@ export class V1_PureGraphManager extends AbstractPureGraphManager { V1_MappingModelCoverageAnalysisResult, input as PlainObject, ), + this, mapping, ); } diff --git a/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/V1_EngineHelper.ts b/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/V1_EngineHelper.ts index b341b2f8f4..b7617df245 100644 --- a/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/V1_EngineHelper.ts +++ b/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/V1_EngineHelper.ts @@ -32,7 +32,6 @@ import { V1_QueryParameterValue, } from './query/V1_Query.js'; import type { PureModel } from '../../../../../graph/PureModel.js'; -import { PackageableElementExplicitReference } from '../../../../../graph/metamodel/pure/packageableElements/PackageableElementReference.js'; import { DEPRECATED__ServiceTestResult } from '../../../../../graph-manager/action/service/DEPRECATED__ServiceTestResult.js'; import type { V1_DEPRECATED__ServiceTestResult } from './service/V1_DEPRECATED__ServiceTestResult.js'; import type { V1_ServiceRegistrationResult } from './service/V1_ServiceRegistrationResult.js'; @@ -130,22 +129,8 @@ export const V1_buildQuery = ( protocol.artifactId, `Query 'artifactId' field is missing`, ); - metamodel.mapping = PackageableElementExplicitReference.create( - graph.getMapping( - guaranteeNonNullable( - protocol.mapping, - `Query 'mapping' field is missing`, - ), - ), - ); - metamodel.runtime = PackageableElementExplicitReference.create( - graph.getRuntime( - guaranteeNonNullable( - protocol.runtime, - `Query 'runtime' field is missing`, - ), - ), - ); + metamodel.mapping = protocol.mapping; + metamodel.runtime = protocol.runtime; metamodel.content = guaranteeNonNullable( protocol.content, `Query 'content' field is missing`, @@ -210,8 +195,8 @@ export const V1_transformQuery = (metamodel: Query): V1_Query => { protocol.versionId = metamodel.versionId; protocol.groupId = metamodel.groupId; protocol.artifactId = metamodel.artifactId; - protocol.mapping = metamodel.mapping.valueForSerialization ?? ''; - protocol.runtime = metamodel.runtime.valueForSerialization ?? ''; + protocol.mapping = metamodel.mapping; + protocol.runtime = metamodel.runtime; protocol.content = metamodel.content; protocol.owner = metamodel.owner; protocol.taggedValues = metamodel.taggedValues?.map((_taggedValue) => { diff --git a/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/analytics/V1_MappingModelCoverageAnalysis.ts b/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/analytics/V1_MappingModelCoverageAnalysis.ts index 52228332c6..6a45611f23 100644 --- a/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/analytics/V1_MappingModelCoverageAnalysis.ts +++ b/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/analytics/V1_MappingModelCoverageAnalysis.ts @@ -33,11 +33,18 @@ import { EntityMappedProperty, EnumMappedProperty, MappedEntity, + MappedEntityInfo, MappedProperty, MappingModelCoverageAnalysisResult, } from '../../../../../../graph-manager/action/analytics/MappingModelCoverageAnalysis.js'; import type { V1_PureModelContext } from '../../model/context/V1_PureModelContext.js'; -import { V1_pureModelContextPropSchema } from '../../transformation/pureProtocol/V1_PureProtocolSerialization.js'; +import { + V1_deserializePureModelContextData, + V1_pureModelContextPropSchema, + V1_serializePureModelContextData, +} from '../../transformation/pureProtocol/V1_PureProtocolSerialization.js'; +import type { V1_PureModelContextData } from '../../model/context/V1_PureModelContextData.js'; +import type { V1_PureGraphManager } from '../../V1_PureGraphManager.js'; enum V1_MappedPropertyType { ENUM = 'enum', @@ -101,13 +108,31 @@ const V1_deserializeMappedProperty = ( } }; +class V1_MappedEntityInfo { + classPath!: string; + isRootEntity!: boolean; + subClasses: string[] = []; + + static readonly serialization = new SerializationFactory( + createModelSchema(V1_MappedEntityInfo, { + classPath: primitive(), + isRootEntity: primitive(), + subClasses: list(primitive()), + }), + ); +} + class V1_MappedEntity { path!: string; properties: V1_MappedProperty[] = []; + info?: V1_MappedEntityInfo | undefined; static readonly serialization = new SerializationFactory( createModelSchema(V1_MappedEntity, { path: primitive(), + info: optional( + usingModelSchema(V1_MappedEntityInfo.serialization.schema), + ), properties: list( custom( (prop) => V1_serializeMappedProperty(prop), @@ -134,12 +159,19 @@ export class V1_MappingModelCoverageAnalysisInput { export class V1_MappingModelCoverageAnalysisResult { mappedEntities: V1_MappedEntity[] = []; + model?: V1_PureModelContextData | undefined; static readonly serialization = new SerializationFactory( createModelSchema(V1_MappingModelCoverageAnalysisResult, { mappedEntities: list( usingModelSchema(V1_MappedEntity.serialization.schema), ), + model: optional( + custom( + (val) => V1_serializePureModelContextData(val), + (val) => V1_deserializePureModelContextData(val), + ), + ), }), ); } @@ -155,17 +187,35 @@ const buildMappedProperty = (protocol: V1_MappedProperty): MappedProperty => ? new EnumMappedProperty(protocol.name, protocol.enumPath) : new MappedProperty(protocol.name); -const buildMappedEntity = (protocol: V1_MappedEntity): MappedEntity => - new MappedEntity( +const buildMappedEntityInfo = ( + protocol: V1_MappedEntityInfo, +): MappedEntityInfo => + new MappedEntityInfo( + protocol.classPath, + protocol.isRootEntity, + protocol.subClasses, + ); + +const buildMappedEntity = (protocol: V1_MappedEntity): MappedEntity => { + const info = protocol.info ? buildMappedEntityInfo(protocol.info) : undefined; + return new MappedEntity( protocol.path, protocol.properties.map((p) => buildMappedProperty(p)), + info, ); +}; export const V1_buildModelCoverageAnalysisResult = ( protocol: V1_MappingModelCoverageAnalysisResult, + graphManager: V1_PureGraphManager, mapping: Mapping, -): MappingModelCoverageAnalysisResult => - new MappingModelCoverageAnalysisResult( +): MappingModelCoverageAnalysisResult => { + const entities = protocol.model?.elements.map((el) => + graphManager.elementProtocolToEntity(el), + ); + return new MappingModelCoverageAnalysisResult( protocol.mappedEntities.map((p) => buildMappedEntity(p)), mapping, + entities, ); +}; diff --git a/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/query/V1_Query.ts b/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/query/V1_Query.ts index 3d2e9bbd28..94e21fc17f 100644 --- a/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/query/V1_Query.ts +++ b/packages/legend-graph/src/graph-manager/protocol/pure/v1/engine/query/V1_Query.ts @@ -87,6 +87,8 @@ export class V1_LightQuery { artifactId!: string; versionId!: string; lastUpdatedAt?: number | undefined; + mapping?: string | undefined; + taggedValues?: V1_TaggedValue[] | undefined; static readonly serialization = new SerializationFactory( createModelSchema(V1_Query, { @@ -94,8 +96,10 @@ export class V1_LightQuery { id: primitive(), groupId: primitive(), lastUpdatedAt: optional(primitive()), + mapping: optional(primitive()), name: primitive(), owner: optional(primitive()), + taggedValues: optional(list(usingModelSchema(V1_taggedValueModelSchema))), versionId: primitive(), }), {