From 4d5719eefdf2a8ebae6a674ea84df7f17b16dc07 Mon Sep 17 00:00:00 2001 From: manelcecs Date: Wed, 10 Jan 2024 19:37:22 +0100 Subject: [PATCH] Temp: performance --- src/domain-services/flows/flow-service.ts | 27 +++++++- src/domain-services/flows/model.ts | 4 +- ...-flow-category-conditions-strategy-impl.ts | 60 ------------------ .../search-flow-by-filters-strategy-impl.ts | 61 ++++++++++++------- .../flows/strategy/impl/utils.ts | 25 ++++++++ 5 files changed, 93 insertions(+), 84 deletions(-) diff --git a/src/domain-services/flows/flow-service.ts b/src/domain-services/flows/flow-service.ts index ca3f6982..f512a842 100644 --- a/src/domain-services/flows/flow-service.ts +++ b/src/domain-services/flows/flow-service.ts @@ -4,6 +4,7 @@ import { Service } from 'typedi'; import { type FlowObjectType } from '../flow-object/model'; import { GetFlowsArgs, UniqueFlowEntity, type FlowOrderBy } from './model'; import { + applySearchFilters, mapFlowOrderBy, removeDuplicatesUniqueFlowEntities, } from './strategy/impl/utils'; @@ -15,7 +16,7 @@ export class FlowService { async getFlows(args: GetFlowsArgs) { const { models, conditions, offset, orderBy, limit } = args; - return await models.flow.find({ + return await models!.flow.find({ orderBy, limit, where: conditions, @@ -23,6 +24,30 @@ export class FlowService { }); } + async getFlowsAsUniqueFlowEntity(args: GetFlowsArgs): Promise { + const { databaseConnection, orderBy, conditions } = args; + + let query = databaseConnection!.queryBuilder() + .distinct('id', 'versionID', orderBy.column) // Include orderBy.column in the distinct selection + .select('id', 'versionID') + .from('flow') + .whereNull('deletedAt') + .orderBy(orderBy.column, orderBy.order); + + if(conditions) { + query = applySearchFilters(query, conditions); + } + + const flows = await query; + + const mapFlowsToUniqueFlowEntities = flows.map((flow) => ({ + id: flow.id, + versionID: flow.versionID, + })); + + return mapFlowsToUniqueFlowEntities; + } + async getFlowsCount(models: Database, conditions: any) { return await models.flow.count({ where: conditions }); } diff --git a/src/domain-services/flows/model.ts b/src/domain-services/flows/model.ts index b253fb62..024fe175 100644 --- a/src/domain-services/flows/model.ts +++ b/src/domain-services/flows/model.ts @@ -2,6 +2,7 @@ import { type Database } from '@unocha/hpc-api-core/src/db'; import { type FlowId } from '@unocha/hpc-api-core/src/db/models/flow'; import { type InstanceDataOfModel } from '@unocha/hpc-api-core/src/db/util/raw-model'; import { type SortOrder } from '../../utils/graphql/pagination'; +import Knex from 'knex'; export type FlowEntity = InstanceDataOfModel; @@ -21,7 +22,8 @@ export type FlowOrderBy = { export type FlowNestedDirection = 'source' | 'destination'; export type GetFlowsArgs = { - models: Database; + models?: Database; + databaseConnection?: Knex; conditions: any; offset?: number; orderBy?: any; diff --git a/src/domain-services/flows/strategy/impl/get-flowIds-flow-category-conditions-strategy-impl.ts b/src/domain-services/flows/strategy/impl/get-flowIds-flow-category-conditions-strategy-impl.ts index 36f437bf..7624c42e 100644 --- a/src/domain-services/flows/strategy/impl/get-flowIds-flow-category-conditions-strategy-impl.ts +++ b/src/domain-services/flows/strategy/impl/get-flowIds-flow-category-conditions-strategy-impl.ts @@ -104,7 +104,6 @@ export class GetFlowIdsFromCategoryConditionsStrategyImpl } const flows = await query; - console.log('flows', flows.length); const mapFlows: UniqueFlowEntity[] = flows.map( (flow) => ({ @@ -114,65 +113,6 @@ export class GetFlowIdsFromCategoryConditionsStrategyImpl ); return { flows: mapFlows }; - // const whereClauseCategoryRef = { - // [Cond.OR]: [ - // categoriesIds.length > 0 - // ? { - // [Cond.AND]: [ - // { - // categoryID: { - // [Op.IN]: categoriesIds, - // }, - // }, - // { objectType: 'flow' }, - // ], - // } - // : {}, - // categoriesIdsFromShortcutFilterIN.length > 0 - // ? { - // [Cond.AND]: [ - // { - // categoryID: { - // [Op.IN]: categoriesIdsFromShortcutFilterIN, - // }, - // }, - // { objectType: 'flow' }, - // ], - // } - // : {}, - // categoriesIdsFromShortcutFilterNOTIN.length > 0 - // ? { - // [Cond.AND]: [ - // { - // categoryID: { - // [Op.NOT_IN]: categoriesIdsFromShortcutFilterNOTIN, - // }, - // }, - // { objectType: 'flow' }, - // ], - // } - // : {}, - // ], - // }; - - // const categoryRefs = await this.categoryService.findCategoryRefs( - // models, - // whereClauseCategoryRef - // ); - - // Map category refs to flow IDs - // keep only unique values - // and return the list of flow IDs - // const mapFlowsToUniqueFlowEntities = categoryRefs.map((categoryRef) => { - // return { - // id: categoryRef.objectID, - // versionID: categoryRef.versionID, - // } as UniqueFlowEntity; - // }) - - // const flows = removeDuplicatesUniqueFlowEntities(mapFlowsToUniqueFlowEntities); - - // return { flows }; } generateWhereClause( diff --git a/src/domain-services/flows/strategy/impl/search-flow-by-filters-strategy-impl.ts b/src/domain-services/flows/strategy/impl/search-flow-by-filters-strategy-impl.ts index 0ba3cb69..ea65ce14 100644 --- a/src/domain-services/flows/strategy/impl/search-flow-by-filters-strategy-impl.ts +++ b/src/domain-services/flows/strategy/impl/search-flow-by-filters-strategy-impl.ts @@ -1,8 +1,7 @@ -import { type FlowId } from '@unocha/hpc-api-core/src/db/models/flow'; -import { Cond, Op } from '@unocha/hpc-api-core/src/db/util/conditions'; +import { Cond } from '@unocha/hpc-api-core/src/db/util/conditions'; import { Service } from 'typedi'; import { FlowService } from '../../flow-service'; -import { UniqueFlowEntity } from '../../model'; +import { FlowEntity, UniqueFlowEntity } from '../../model'; import { type FlowSearchArgs, type FlowSearchStrategy, @@ -13,10 +12,8 @@ import { GetFlowIdsFromCategoryConditionsStrategyImpl } from './get-flowIds-flow import { GetFlowIdsFromObjectConditionsStrategyImpl } from './get-flowIds-flow-object-conditions-strategy-impl'; import { intersectUniqueFlowEntities, - mapCountResultToCountObject, mapFlowObjectConditions, mapFlowOrderBy, - mergeUniqueEntities, prepareFlowConditions, sortEntitiesByReferenceList, } from './utils'; @@ -63,30 +60,32 @@ export class SearchFlowByFiltersStrategy implements FlowSearchStrategy { // In this case we fetch the list of flows from the database // using the orderBy // We can also filter by flowFilters - const flowConditions = prepareFlowConditions(flowFilters); + const orderByForFlow = mapFlowOrderBy(orderBy); - const flowsToSort = await this.flowService.getFlows({ - models, - conditions: flowConditions, - orderBy: { column: orderBy.column, order: orderBy.order }, + const flowsToSort: UniqueFlowEntity[] = await this.flowService.getFlowsAsUniqueFlowEntity({ + databaseConnection, + conditions: flowFilters, + orderBy: orderByForFlow, }); - const flowIDsFromSortingEntity: UniqueFlowEntity[] = flowsToSort.map( - (flow) => ({ id: flow.id, versionID: flow.versionID }) - ); // Since there can be many flowIDs returned // This can cause 'Maximum call stack size exceeded' error // When using the spread operator - a workaround is to use push fot each element - for (const flow of flowIDsFromSortingEntity) { - sortByFlowIDs.push(flow); + // also, we need to map the FlowEntity to UniqueFlowEntity + for (const flow of flowsToSort) { + const uniqueFlowEntity: UniqueFlowEntity = { + id: flow.id, + versionID: flow.versionID, + }; + sortByFlowIDs.push(uniqueFlowEntity); } } // Now we need to check if we need to filter by category // if it's using any of the shorcuts // or if there are any flowCategoryFilters - const isSearchByCategoryShotcut = shortcutFilter !== null; - + const isSearchByCategoryShotcut = shortcutFilter !== null && shortcutFilter.length > 0; + const isFilterByCategory = isSearchByCategoryShotcut || flowCategoryFilters?.length > 0; @@ -101,6 +100,7 @@ export class SearchFlowByFiltersStrategy implements FlowSearchStrategy { shortcutFilter, flowObjectsConditions: undefined, }); + // Since there can be many flowIDs returned // This can cause 'Maximum call stack size exceeded' error // When using the spread operator - a workaround is to use push fot each element @@ -123,7 +123,13 @@ export class SearchFlowByFiltersStrategy implements FlowSearchStrategy { models, flowObjectsConditions: flowObjectConditionsMap, }); - flowsFromObjectFilters.push(...flows); + + // Since there can be many flowIDs returned + // This can cause 'Maximum call stack size exceeded' error + // When using the spread operator - a workaround is to use push fot each element + for (const flow of flows) { + flowsFromObjectFilters.push(flow); + } } // Lastly, we need to check if we need to filter by flow @@ -134,13 +140,22 @@ export class SearchFlowByFiltersStrategy implements FlowSearchStrategy { const flowsFromFlowFilters: UniqueFlowEntity[] = []; if (isSortByEntity && isFilterByFlow) { const flowConditions = prepareFlowConditions(flowFilters); - const flows = await this.flowService.getFlows({ + const flows: FlowEntity[] = await this.flowService.getFlows({ models, conditions: flowConditions, orderBy: { column: orderBy.column, order: orderBy.order }, }); + + // Since there can be many flowIDs returned + // This can cause 'Maximum call stack size exceeded' error + // When using the spread operator - a workaround is to use push fot each element + // also, we need to map the FlowEntity to UniqueFlowEntity for (const flow of flows) { - flowsFromFlowFilters.push({ id: flow.id, versionID: flow.versionID }); + const uniqueFlowEntity: UniqueFlowEntity = { + id: flow.id, + versionID: flow.versionID, + }; + sortByFlowIDs.push(uniqueFlowEntity); } } @@ -165,6 +180,7 @@ export class SearchFlowByFiltersStrategy implements FlowSearchStrategy { sortByFlowIDs ); + // Then we are going to slice the flows using the limit and offset const reducedFlows: UniqueFlowEntity[] = sortedFlows.slice( offset, @@ -174,11 +190,12 @@ export class SearchFlowByFiltersStrategy implements FlowSearchStrategy { // Once the list of elements is reduced, we need to build the conditions const searchConditions = this.buildConditions(reducedFlows); + const orderByForFlow = mapFlowOrderBy(orderBy); + const flows = await this.flowService.getFlows({ models, conditions: searchConditions, - limit, - orderBy: { column: orderBy.column, order: orderBy.order }, + orderBy: orderByForFlow, }); return { flows, count }; diff --git a/src/domain-services/flows/strategy/impl/utils.ts b/src/domain-services/flows/strategy/impl/utils.ts index 3231cc8e..72c8603a 100644 --- a/src/domain-services/flows/strategy/impl/utils.ts +++ b/src/domain-services/flows/strategy/impl/utils.ts @@ -6,6 +6,7 @@ import { type SearchFlowsFilters, } from '../../graphql/args'; import { UniqueFlowEntity } from '../../model'; +import Knex from 'knex'; /* * Map structure: @@ -304,3 +305,27 @@ export function removeDuplicatesUniqueFlowEntities( return Array.from(uniqueEntities.values()); } + +export function applySearchFilters(query: Knex.QueryBuilder, filters: SearchFlowsFilters): Knex.QueryBuilder { + // Check if 'id' filter is defined and apply it + if (filters.id !== null && filters.id !== undefined) { + query.whereIn('id', filters.id); + } + + // Check if 'activeStatus' filter is defined and apply it + if (filters.activeStatus !== null && filters.activeStatus !== undefined) { + query.andWhere('activeStatus', filters.activeStatus); + } + + // Check if 'amountUSD' filter is defined and apply it + if (filters.amountUSD !== null && filters.amountUSD !== undefined) { + query.andWhere('amountUSD', filters.amountUSD); + } + + // Check if 'restricted' filter is defined and apply it + if (filters.restricted !== null && filters.restricted !== undefined) { + query.andWhere('restricted', filters.restricted); + } + + return query; +} \ No newline at end of file