From 5b0d1b7f725cc6a109847eee876d764d5248d7dc Mon Sep 17 00:00:00 2001 From: manelcecs Date: Wed, 24 Apr 2024 12:33:15 +0200 Subject: [PATCH] Add 'search-flow-by-filters-strategy' implementation Nested strategies to find flowIDs from different filters --- .../flows/strategy/flowID-search-strategy.ts | 23 ++ ...-flow-category-conditions-strategy-impl.ts | 120 +++++++ ...-from-nested-flow-filters-strategy-impl.ts | 113 ++++++ ...ds-flow-object-conditions-strategy-impl.ts | 36 ++ .../search-flow-by-filters-strategy-impl.ts | 335 ++++++++++++++++++ 5 files changed, 627 insertions(+) create mode 100644 src/domain-services/flows/strategy/flowID-search-strategy.ts create mode 100644 src/domain-services/flows/strategy/impl/get-flowIds-flow-category-conditions-strategy-impl.ts create mode 100644 src/domain-services/flows/strategy/impl/get-flowIds-flow-from-nested-flow-filters-strategy-impl.ts create mode 100644 src/domain-services/flows/strategy/impl/get-flowIds-flow-object-conditions-strategy-impl.ts create mode 100644 src/domain-services/flows/strategy/impl/search-flow-by-filters-strategy-impl.ts diff --git a/src/domain-services/flows/strategy/flowID-search-strategy.ts b/src/domain-services/flows/strategy/flowID-search-strategy.ts new file mode 100644 index 00000000..20c33104 --- /dev/null +++ b/src/domain-services/flows/strategy/flowID-search-strategy.ts @@ -0,0 +1,23 @@ +import { type Database } from '@unocha/hpc-api-core/src/db'; +import type Knex from 'knex'; +import { type ShortcutCategoryFilter } from '../../categories/model'; +import { type FlowObjectFilterGrouped } from '../../flow-object/model'; +import { type FlowCategory, type NestedFlowFilters } from '../graphql/args'; +import { type UniqueFlowEntity } from '../model'; + +export interface FlowIdSearchStrategyResponse { + flows: UniqueFlowEntity[]; +} + +export interface FlowIdSearchStrategyArgs { + databaseConnection: Knex; + models: Database; + flowObjectFilterGrouped?: FlowObjectFilterGrouped; + flowCategoryConditions?: FlowCategory[]; + nestedFlowFilters?: NestedFlowFilters; + shortcutFilters?: ShortcutCategoryFilter[] | null; +} + +export interface FlowIDSearchStrategy { + search(args: FlowIdSearchStrategyArgs): Promise; +} diff --git a/src/domain-services/flows/strategy/impl/get-flowIds-flow-category-conditions-strategy-impl.ts b/src/domain-services/flows/strategy/impl/get-flowIds-flow-category-conditions-strategy-impl.ts new file mode 100644 index 00000000..1ec367e1 --- /dev/null +++ b/src/domain-services/flows/strategy/impl/get-flowIds-flow-category-conditions-strategy-impl.ts @@ -0,0 +1,120 @@ +import { type CategoryId } from '@unocha/hpc-api-core/src/db/models/category'; +import { Op } from '@unocha/hpc-api-core/src/db/util/conditions'; +import { createBrandedValue } from '@unocha/hpc-api-core/src/util/types'; +import { Service } from 'typedi'; +import { CategoryService } from '../../../categories/category-service'; +import { type UniqueFlowEntity } from '../../model'; +import { + type FlowIDSearchStrategy, + type FlowIdSearchStrategyArgs, + type FlowIdSearchStrategyResponse, +} from '../flowID-search-strategy'; +import { mapFlowCategoryConditionsToWhereClause } from './utils'; + +@Service() +export class GetFlowIdsFromCategoryConditionsStrategyImpl + implements FlowIDSearchStrategy +{ + constructor(private readonly categoryService: CategoryService) {} + + async search( + args: FlowIdSearchStrategyArgs + ): Promise { + const { + models, + flowCategoryConditions, + shortcutFilters, + databaseConnection, + } = args; + + let categoriesIds: CategoryId[] = []; + + let whereClause = null; + if (flowCategoryConditions) { + whereClause = mapFlowCategoryConditionsToWhereClause( + flowCategoryConditions + ); + } + if (whereClause) { + const categories = await this.categoryService.findCategories( + models, + whereClause + ); + + categoriesIds = categories.map((category) => category.id); + } + + // Add category IDs from shortcut filter + // to the list of category IDs IN or NOT_IN + const categoriesIdsFromShortcutFilterIN: CategoryId[] = []; + const categoriesIdsFromShortcutFilterNOTIN: CategoryId[] = []; + + if (shortcutFilters) { + for (const shortcut of shortcutFilters) { + if (shortcut.operation === Op.IN) { + categoriesIdsFromShortcutFilterIN.push( + createBrandedValue(shortcut.id) + ); + } else { + categoriesIdsFromShortcutFilterNOTIN.push( + createBrandedValue(shortcut.id) + ); + } + } + } + + let joinQuery = databaseConnection! + .queryBuilder() + .distinct('flow.id', 'flow.versionID') + .from('flow') + .where('flow.deletedAt', null) + .join('categoryRef', function () { + this.on('flow.id', '=', 'categoryRef.objectID').andOn( + 'flow.versionID', + '=', + 'categoryRef.versionID' + ); + }); + + if (categoriesIds.length > 0) { + joinQuery = joinQuery.andWhere(function () { + this.where('categoryRef.categoryID', 'IN', categoriesIds).andWhere( + 'categoryRef.objectType', + 'flow' + ); + }); + } + + if (categoriesIdsFromShortcutFilterIN.length > 0) { + joinQuery = joinQuery.andWhere(function () { + this.where( + 'categoryRef.categoryID', + 'IN', + categoriesIdsFromShortcutFilterIN + ).andWhere('categoryRef.objectType', 'flow'); + }); + } + + if (categoriesIdsFromShortcutFilterNOTIN.length > 0) { + joinQuery = joinQuery.andWhere(function () { + this.where( + 'categoryRef.categoryID', + 'NOT IN', + categoriesIdsFromShortcutFilterNOTIN + ).andWhere('categoryRef.objectType', 'flow'); + }); + } + + const flows = await joinQuery; + + const mapFlows: UniqueFlowEntity[] = flows.map( + (flow) => + ({ + id: flow.id, + versionID: flow.versionID, + }) satisfies UniqueFlowEntity + ); + + return { flows: mapFlows }; + } +} diff --git a/src/domain-services/flows/strategy/impl/get-flowIds-flow-from-nested-flow-filters-strategy-impl.ts b/src/domain-services/flows/strategy/impl/get-flowIds-flow-from-nested-flow-filters-strategy-impl.ts new file mode 100644 index 00000000..3cc15e42 --- /dev/null +++ b/src/domain-services/flows/strategy/impl/get-flowIds-flow-from-nested-flow-filters-strategy-impl.ts @@ -0,0 +1,113 @@ +import { Service } from 'typedi'; +import { ExternalReferenceService } from '../../../external-reference/external-reference-service'; +import { LegacyService } from '../../../legacy/legacy-service'; +import { ReportDetailService } from '../../../report-details/report-detail-service'; +import { FlowService } from '../../flow-service'; +import type { IGetUniqueFlowsArgs, UniqueFlowEntity } from '../../model'; +import { + type FlowIDSearchStrategy, + type FlowIdSearchStrategyArgs, + type FlowIdSearchStrategyResponse, +} from '../flowID-search-strategy'; +import { + buildSearchFlowsConditions, + defaultFlowOrderBy, + intersectUniqueFlowEntities, +} from './utils'; + +@Service() +export class GetFlowIdsFromNestedFlowFiltersStrategyImpl + implements FlowIDSearchStrategy +{ + constructor( + private readonly reportDetailService: ReportDetailService, + private readonly legacyService: LegacyService, + private readonly externalRefenceService: ExternalReferenceService, + private readonly flowService: FlowService + ) {} + + async search( + args: FlowIdSearchStrategyArgs + ): Promise { + const { databaseConnection, models, nestedFlowFilters } = args; + + let flowsReporterReferenceCode: UniqueFlowEntity[] = []; + let flowsSourceSystemId: UniqueFlowEntity[] = []; + let flowsSystemId: UniqueFlowEntity[] = []; + const flowsLegacyId: UniqueFlowEntity[] = []; + + // Get the flowIDs using 'reporterReferenceCode' + if (nestedFlowFilters?.reporterRefCode) { + flowsReporterReferenceCode = + await this.reportDetailService.getUniqueFlowIDsFromReportDetailsByReporterReferenceCode( + models, + nestedFlowFilters.reporterRefCode + ); + } + + // Get the flowIDs using 'sourceSystemID' from 'reportDetail' + if (nestedFlowFilters?.sourceSystemID) { + flowsSourceSystemId = + await this.reportDetailService.getUniqueFlowIDsFromReportDetailsBySourceSystemID( + models, + nestedFlowFilters.sourceSystemID + ); + } + + // Get the flowIDs using 'systemID' from 'externalRefecence' + if (nestedFlowFilters?.systemID) { + flowsSystemId = + await this.externalRefenceService.getUniqueFlowIDsBySystemID( + models, + nestedFlowFilters.systemID + ); + } + + // Get the flowIDs using 'legacyID' + if (nestedFlowFilters?.legacyID) { + const flowID = await this.legacyService.getFlowIdFromLegacyId( + models, + nestedFlowFilters.legacyID + ); + + if (flowID) { + flowsLegacyId.push({ + id: flowID, + versionID: 1, + }); + } + } + + // Intersect the flowIDs from the nestedFlowFilters + const flowIDsFromNestedFlowFilters: UniqueFlowEntity[] = + intersectUniqueFlowEntities( + flowsReporterReferenceCode, + flowsSourceSystemId, + flowsSystemId, + flowsLegacyId + ); + + if (flowIDsFromNestedFlowFilters.length === 0) { + return { flows: [] }; + } + // Once gathered and disjoined the flowIDs from the nestedFlowFilters + // Look after this uniqueFlows in the flow table + // To verify the flow is not deleted + const uniqueFlowEntitiesNotDeleted = []; + + // Slice the flowIDs in chunks of 1000 to avoid the SQL query limit + for (let i = 0; i < flowIDsFromNestedFlowFilters.length; i += 1000) { + const getFlowArgs: IGetUniqueFlowsArgs = { + databaseConnection, + orderBy: defaultFlowOrderBy(), + whereClauses: buildSearchFlowsConditions( + flowIDsFromNestedFlowFilters.slice(i, i + 1000) + ), + }; + const uniqueFlowsNotDeleted = + await this.flowService.getFlowsAsUniqueFlowEntity(getFlowArgs); + uniqueFlowEntitiesNotDeleted.push(...uniqueFlowsNotDeleted); + } + return { flows: uniqueFlowEntitiesNotDeleted }; + } +} diff --git a/src/domain-services/flows/strategy/impl/get-flowIds-flow-object-conditions-strategy-impl.ts b/src/domain-services/flows/strategy/impl/get-flowIds-flow-object-conditions-strategy-impl.ts new file mode 100644 index 00000000..58bb02e8 --- /dev/null +++ b/src/domain-services/flows/strategy/impl/get-flowIds-flow-object-conditions-strategy-impl.ts @@ -0,0 +1,36 @@ +import { Service } from 'typedi'; +import { FlowObjectService } from '../../../flow-object/flow-object-service'; +import { FlowService } from '../../flow-service'; +import { + type FlowIDSearchStrategy, + type FlowIdSearchStrategyArgs, + type FlowIdSearchStrategyResponse, +} from '../flowID-search-strategy'; + +@Service() +export class GetFlowIdsFromObjectConditionsStrategyImpl + implements FlowIDSearchStrategy +{ + constructor( + private readonly flowObjectService: FlowObjectService, + private readonly flowService: FlowService + ) {} + + async search( + args: FlowIdSearchStrategyArgs + ): Promise { + const { flowObjectFilterGrouped, databaseConnection } = args; + + if (!flowObjectFilterGrouped) { + return { flows: [] }; + } + + const flowObjects = + await this.flowObjectService.getFlowObjectsByFlowObjectConditions( + databaseConnection, + flowObjectFilterGrouped + ); + + return { flows: flowObjects }; + } +} diff --git a/src/domain-services/flows/strategy/impl/search-flow-by-filters-strategy-impl.ts b/src/domain-services/flows/strategy/impl/search-flow-by-filters-strategy-impl.ts new file mode 100644 index 00000000..47f1587f --- /dev/null +++ b/src/domain-services/flows/strategy/impl/search-flow-by-filters-strategy-impl.ts @@ -0,0 +1,335 @@ +import type Knex from 'knex'; +import { Service } from 'typedi'; +import { FlowService } from '../../flow-service'; +import type { + FlowInstance, + FlowOrderByCond, + UniqueFlowEntity, +} from '../../model'; +import type { + FlowSearchArgs, + FlowSearchStrategy, + FlowSearchStrategyResponse, +} from '../flow-search-strategy'; +import { type FlowIdSearchStrategyResponse } from '../flowID-search-strategy'; +import { GetFlowIdsFromCategoryConditionsStrategyImpl } from './get-flowIds-flow-category-conditions-strategy-impl'; +import { GetFlowIdsFromNestedFlowFiltersStrategyImpl } from './get-flowIds-flow-from-nested-flow-filters-strategy-impl'; +import { GetFlowIdsFromObjectConditionsStrategyImpl } from './get-flowIds-flow-object-conditions-strategy-impl'; +import { + buildOrderByReference, + buildSearchFlowsConditions, + defaultFlowOrderBy, + intersectUniqueFlowEntities, + mapFlowFiltersToFlowObjectFiltersGrouped, + mapFlowOrderBy, + mergeUniqueEntities, + prepareFlowConditions, + prepareFlowStatusConditions, +} from './utils'; + +@Service() +export class SearchFlowByFiltersStrategy implements FlowSearchStrategy { + constructor( + private readonly flowService: FlowService, + private readonly getFlowIdsFromCategoryConditions: GetFlowIdsFromCategoryConditionsStrategyImpl, + private readonly getFlowIdsFromObjectConditions: GetFlowIdsFromObjectConditionsStrategyImpl, + private readonly getFlowIdsFromNestedFlowFilters: GetFlowIdsFromNestedFlowFiltersStrategyImpl + ) {} + + async search(args: FlowSearchArgs): Promise { + const { + models, + databaseConnection, + flowFilters, + flowObjectFilters, + flowCategoryFilters, + nestedFlowFilters, + limit, + offset, + shortcutFilters, + statusFilter, + orderBy, + shouldIncludeChildrenOfParkedFlows, + } = args; + + // First, we need to check if we need to sort by a certain entity + // and if so, we need to map the orderBy to be from that entity + // obtain the entities relation to the flow + // to be able to sort the flows using the entity + const isSortByEntity = orderBy && orderBy.entity !== 'flow'; + const sortByFlowIDs: UniqueFlowEntity[] = []; + const orderByForFlow = mapFlowOrderBy(orderBy); + + if (isSortByEntity) { + // Get the flowIDs using the orderBy entity + const flowIDsFromSortingEntity: UniqueFlowEntity[] = + await this.flowService.getFlowIDsFromEntity( + databaseConnection, + orderBy + ); + // Since there can be many flowIDs returned + // This can cause 'Maximum call stack size exceeded' error + // When using the spread operator - a workaround is to use push fot each element + // also, we need to map the FlowEntity to UniqueFlowEntity + for (const uniqueFlow of flowIDsFromSortingEntity) { + sortByFlowIDs.push(uniqueFlow); + } + } else { + // In this case we fetch the list of flows from the database + // using the orderBy + const flowsToSort: UniqueFlowEntity[] = + await this.flowService.getFlowsAsUniqueFlowEntity({ + databaseConnection, + orderBy: orderByForFlow, + }); + + // Since there can be many flowIDs returned + // This can cause 'Maximum call stack size exceeded' error + // When using the spread operator - a workaround is to use push fot each element + // also, we need to map the FlowEntity to UniqueFlowEntity + for (const flow of flowsToSort) { + sortByFlowIDs.push(flow); + } + } + + // We need to fetch the flowIDs by the nestedFlowFilters + // if there are any + const isFilterByNestedFilters = nestedFlowFilters !== undefined; + const flowIDsFromNestedFlowFilters: UniqueFlowEntity[] = []; + + if (isFilterByNestedFilters) { + const { flows }: FlowIdSearchStrategyResponse = + await this.getFlowIdsFromNestedFlowFilters.search({ + databaseConnection, + models, + nestedFlowFilters, + }); + + // If after this filter we have no flows, we can return an empty array + if (flows.length === 0) { + return { flows: [], count: 0 }; + } + // Since there can be many flowIDs returned + // This can cause 'Maximum call stack size exceeded' error + // When using the spread operator - a workaround is to use push fot each element + for (const flow of flows) { + flowIDsFromNestedFlowFilters.push(flow); + } + } + + // Now we need to check if we need to filter by category + // if it's using any of the shorcuts + // or if there are any flowCategoryFilters + const isSearchByCategoryShotcut = + shortcutFilters !== null && shortcutFilters.length > 0; + + const isFilterByCategory = + isSearchByCategoryShotcut || flowCategoryFilters?.length > 0; + + const flowsFromCategoryFilters: UniqueFlowEntity[] = []; + + if (isFilterByCategory) { + const { flows }: FlowIdSearchStrategyResponse = + await this.getFlowIdsFromCategoryConditions.search({ + databaseConnection, + models, + flowCategoryConditions: flowCategoryFilters ?? [], + shortcutFilters, + }); + + // If after this filter we have no flows, we can return an empty array + if (flows.length === 0) { + return { flows: [], count: 0 }; + } + + // Since there can be many flowIDs returned + // This can cause 'Maximum call stack size exceeded' error + // When using the spread operator - a workaround is to use push fot each element + for (const flow of flows) { + flowsFromCategoryFilters.push(flow); + } + } + + // After that, if we need to filter by flowObjects + // Obtain the flowIDs from the flowObjects + const isFilterByFlowObjects = flowObjectFilters?.length > 0; + + const flowsFromObjectFilters: UniqueFlowEntity[] = []; + if (isFilterByFlowObjects) { + // Firts step is to map the filters to the FlowObjectFiltersGrouped + // To allow doing inclusive filtering between filters of the same type+direction + // But exclusive filtering between filters of different type+direction + const flowObjectFiltersGrouped = + mapFlowFiltersToFlowObjectFiltersGrouped(flowObjectFilters); + + const { flows }: FlowIdSearchStrategyResponse = + await this.getFlowIdsFromObjectConditions.search({ + databaseConnection, + models, + flowObjectFilterGrouped: flowObjectFiltersGrouped, + }); + + // If after this filter we have no flows, we can return an empty array + if (flows.length === 0) { + return { flows: [], count: 0 }; + } + + // Since there can be many flowIDs returned + // This can cause 'Maximum call stack size exceeded' error + // When using the spread operator - a workaround is to use push fot each element + for (const flow of flows) { + flowsFromObjectFilters.push(flow); + } + + // If 'includeChildrenOfParkedFlows' is defined and true + // we need to obtain the flowIDs from the childs whose parent flows are parked + if (shouldIncludeChildrenOfParkedFlows) { + // We need to obtain the flowIDs from the childs whose parent flows are parked + const childs = + await this.flowService.getParkedParentFlowsByFlowObjectFilter( + databaseConnection, + flowObjectFiltersGrouped + ); + + for (const child of childs) { + flowsFromObjectFilters.push(child); + } + } + } + + // Lastly, we need to check if we need to filter by flow + // And if we didn't did it before when sorting by entity + // if so, we need to obtain the flowIDs from the flowFilters + const isFilterByFlow = flowFilters !== undefined; + const isFilterByFlowStatus = statusFilter !== undefined; + + const flowsFromFlowFilters: UniqueFlowEntity[] = []; + if (isFilterByFlow || isFilterByFlowStatus) { + let flowConditions = prepareFlowConditions(flowFilters); + // Add status filter conditions if provided + flowConditions = prepareFlowStatusConditions( + flowConditions, + statusFilter + ); + + const orderByForFlowFilter = isSortByEntity + ? defaultFlowOrderBy() + : orderByForFlow; + + const flows: UniqueFlowEntity[] = + await this.flowService.getFlowsAsUniqueFlowEntity({ + databaseConnection, + conditions: flowConditions, + orderBy: orderByForFlowFilter, + }); + + // If after this filter we have no flows, we can return an empty array + if (flows.length === 0) { + return { flows: [], count: 0 }; + } + + // Since there can be many flowIDs returned + // This can cause 'Maximum call stack size exceeded' error + // When using the spread operator - a workaround is to use push fot each element + // also, we need to map the FlowEntity to UniqueFlowEntity + for (const flow of flows) { + flowsFromFlowFilters.push(flow); + } + } + + // We need to intersect the flowIDs from the flowObjects, flowCategoryFilters and flowFilters + // to obtain the flowIDs that match all the filters + const deduplicatedFlows: UniqueFlowEntity[] = intersectUniqueFlowEntities( + flowsFromCategoryFilters, + flowsFromObjectFilters, + flowsFromFlowFilters, + flowIDsFromNestedFlowFilters + ); + + if (deduplicatedFlows.length === 0) { + return { flows: [], count: 0 }; + } + + // We are going to sort the deduplicated flows + // using the sortByFlowIDs if there are any + let sortedFlows: UniqueFlowEntity[] = []; + // While sorting we have the same amount or less flows 'sorted' than deduplicatedFlows + // That means we need to keep the sortedFilters and then keep the rest of deduplicatedFlows thar are not in sortedFlows + // If we don't do this it may cause that just changing the orderBy we get different results + // Because we get rid of those flows that are not present in the sortedFlows list + sortedFlows = intersectUniqueFlowEntities(sortByFlowIDs, deduplicatedFlows); + + sortedFlows = mergeUniqueEntities(sortedFlows, deduplicatedFlows); + + const count = sortedFlows.length; + + const flowEntityOrderBy = isSortByEntity + ? buildOrderByReference(sortedFlows) + : orderByForFlow; + + const flows = await this.progresiveSearch( + databaseConnection, + sortedFlows, + limit, + offset ?? 0, + flowEntityOrderBy, + [] + ); + + return { flows, count }; + } + + /** + * This method progressively search the flows + * accumulating the results in the flowResponse + * until the limit is reached or there are no more flows + * in the sortedFlows + * + * Since this is a recursive, the exit condition is when + * the flowResponse length is equal to the limit + * or the reducedFlows length is less than the limit after doing the search + * + * @param models + * @param sortedFlows + * @param limit + * @param offset + * @param orderBy + * @param flowResponse + * @returns list of flows + */ + async progresiveSearch( + databaseConnection: Knex, + sortedFlows: UniqueFlowEntity[], + limit: number, + offset: number, + orderBy: FlowOrderByCond | null, + flowResponse: FlowInstance[] + ): Promise { + const reducedFlows = sortedFlows.slice(offset, offset + limit); + + const whereConditions = buildSearchFlowsConditions(reducedFlows); + + const flows = await this.flowService.getFlowsRaw( + databaseConnection, + whereConditions, + orderBy + ); + + flowResponse.push(...flows); + + if (flowResponse.length === limit || reducedFlows.length < limit) { + return flowResponse; + } + + // Recursive call + offset += limit; + return await this.progresiveSearch( + databaseConnection, + sortedFlows, + limit, + offset, + orderBy, + flowResponse + ); + } +}