diff --git a/src/domain-services/categories/model.ts b/src/domain-services/categories/model.ts index e139940e..60d402c1 100644 --- a/src/domain-services/categories/model.ts +++ b/src/domain-services/categories/model.ts @@ -31,4 +31,5 @@ export type CategoryGroup = export type ShortcutCategoryFilter = { category: string; operation: typeof Op.IN | typeof Op.NOT_IN; + id: number; }; diff --git a/src/domain-services/flows/flow-search-service.ts b/src/domain-services/flows/flow-search-service.ts index 2cc466de..f8f2ccdc 100644 --- a/src/domain-services/flows/flow-search-service.ts +++ b/src/domain-services/flows/flow-search-service.ts @@ -319,14 +319,14 @@ export class FlowSearchService { isStandardFlows: boolean ): ShortcutCategoryFilter[] | null { const filters = [ - { flag: isPendingFlows, category: 'Pending' }, - { flag: isCommitmentFlows, category: 'Commitment' }, - { flag: isPaidFlows, category: 'Paid' }, - { flag: isPledgedFlows, category: 'Pledge' }, - { flag: isCarryoverFlows, category: 'Carryover' }, - { flag: isParkedFlows, category: 'Parked' }, - { flag: isPassThroughFlows, category: 'Pass Through' }, - { flag: isStandardFlows, category: 'Standard' }, + { flag: isPendingFlows, category: 'Pending', id: 45 }, + { flag: isCommitmentFlows, category: 'Commitment', id: 47 }, + { flag: isPaidFlows, category: 'Paid', id: 48 }, + { flag: isPledgedFlows, category: 'Pledge', id: 46 }, + { flag: isCarryoverFlows, category: 'Carryover', id: 137 }, + { flag: isParkedFlows, category: 'Parked', id: 1252 }, + { flag: isPassThroughFlows, category: 'Pass Through', id: 136 }, + { flag: isStandardFlows, category: 'Standard', id: 133 }, ]; const shortcutFilters: ShortcutCategoryFilter[] = filters @@ -334,6 +334,7 @@ export class FlowSearchService { .map((filter) => ({ category: filter.category, operation: filter.flag ? Op.IN : Op.NOT_IN, + id: filter.id, })); return shortcutFilters.length > 0 ? shortcutFilters : null; diff --git a/src/domain-services/flows/flow-service.ts b/src/domain-services/flows/flow-service.ts index 6e14397c..02c44a53 100644 --- a/src/domain-services/flows/flow-service.ts +++ b/src/domain-services/flows/flow-service.ts @@ -43,14 +43,7 @@ export class FlowService { async getFlowsAsUniqueFlowEntity( args: GetFlowsArgs ): Promise { - const { - databaseConnection, - orderBy, - conditions, - offset, - limit, - whereClauses, - } = args; + const { databaseConnection, orderBy, conditions, whereClauses } = args; let query = databaseConnection! .queryBuilder() @@ -59,6 +52,8 @@ export class FlowService { .whereNull('deletedAt'); if (orderBy.raw) { query = query.orderByRaw(orderBy.raw); + } else if (orderBy.entity !== 'flow') { + query = query.orderBy('id', orderBy.order); } else { query = query.orderBy(orderBy.column, orderBy.order); } @@ -70,17 +65,7 @@ export class FlowService { query = query.andWhere(prepareCondition(whereClauses)); } - // Because this list can be really large (+330K entries), we need to split it in chunks - // Using limit and offset as cursors - const databaseParsingLimit = 10_000; - if (limit && offset) { - query = query.limit(databaseParsingLimit + offset).offset(offset); - } else if (limit) { - query = query.limit(limit + databaseParsingLimit); - } - const flows = await query; - const mapFlowsToUniqueFlowEntities = flows.map((flow) => ({ id: flow.id, versionID: flow.versionID, @@ -106,6 +91,8 @@ export class FlowService { .whereNull('deletedAt'); if (orderBy.raw) { query = query.orderByRaw(orderBy.raw); + } else if (orderBy.entity !== 'flow') { + query = query.orderBy('id', orderBy.order); } else { query = query.orderBy(orderBy.column, orderBy.order); } diff --git a/src/domain-services/flows/strategy/impl/get-flowIds-flow-category-conditions-strategy-impl.ts b/src/domain-services/flows/strategy/impl/get-flowIds-flow-category-conditions-strategy-impl.ts index 0f08538c..021df0e1 100644 --- a/src/domain-services/flows/strategy/impl/get-flowIds-flow-category-conditions-strategy-impl.ts +++ b/src/domain-services/flows/strategy/impl/get-flowIds-flow-category-conditions-strategy-impl.ts @@ -17,20 +17,6 @@ export class GetFlowIdsFromCategoryConditionsStrategyImpl { constructor(private readonly categoryService: CategoryService) {} - private readonly categoryIDsMap: Map = new Map< - string, - number - >([ - ['Pending', 45], - ['Pledge', 46], - ['Commitment', 47], - ['Paid', 48], - ['Standard', 133], - ['Pass through', 136], - ['Carryover', 137], - ['Parked', 1252], - ]); - async search( args: FlowIdSearchStrategyArgs ): Promise { @@ -41,19 +27,21 @@ export class GetFlowIdsFromCategoryConditionsStrategyImpl databaseConnection, } = args; - const categoriesIds: CategoryId[] = []; - - const whereClause = mapFlowCategoryConditionsToWhereClause( - flowCategoryConditions! - ); + let categoriesIds: CategoryId[] = []; + let whereClause = null; + if (flowCategoryConditions) { + whereClause = mapFlowCategoryConditionsToWhereClause( + flowCategoryConditions + ); + } if (whereClause) { const categories = await this.categoryService.findCategories( models, whereClause ); - categories.map((category) => category.id); + categoriesIds = categories.map((category) => category.id); } // Add category IDs from shortcut filter @@ -63,17 +51,14 @@ export class GetFlowIdsFromCategoryConditionsStrategyImpl if (shortcutFilter) { for (const shortcut of shortcutFilter) { - const shortcutCategoryID = this.categoryIDsMap.get(shortcut.category); - if (shortcutCategoryID) { - if (shortcut.operation === Op.IN) { - categoriesIdsFromShortcutFilterIN.push( - createBrandedValue(shortcutCategoryID) - ); - } else { - categoriesIdsFromShortcutFilterNOTIN.push( - createBrandedValue(shortcutCategoryID) - ); - } + if (shortcut.operation === Op.IN) { + categoriesIdsFromShortcutFilterIN.push( + createBrandedValue(shortcut.id) + ); + } else { + categoriesIdsFromShortcutFilterNOTIN.push( + createBrandedValue(shortcut.id) + ); } } } @@ -91,7 +76,7 @@ export class GetFlowIdsFromCategoryConditionsStrategyImpl }); if (categoriesIds.length > 0) { - joinQuery = joinQuery.orWhere(function () { + joinQuery = joinQuery.andWhere(function () { this.where('categoryRef.categoryID', 'IN', categoriesIds) .andWhere('categoryRef.objectType', 'flow') .andWhere('flow.deletedAt', null); @@ -99,7 +84,7 @@ export class GetFlowIdsFromCategoryConditionsStrategyImpl } if (categoriesIdsFromShortcutFilterIN.length > 0) { - joinQuery = joinQuery.orWhere(function () { + joinQuery = joinQuery.andWhere(function () { this.where( 'categoryRef.categoryID', 'IN', @@ -111,7 +96,7 @@ export class GetFlowIdsFromCategoryConditionsStrategyImpl } if (categoriesIdsFromShortcutFilterNOTIN.length > 0) { - joinQuery = joinQuery.orWhere(function () { + joinQuery = joinQuery.andWhere(function () { this.where( 'categoryRef.categoryID', 'NOT IN', diff --git a/src/domain-services/flows/strategy/impl/get-flowIds-flow-from-nested-flow-filters-strategy-impl.ts b/src/domain-services/flows/strategy/impl/get-flowIds-flow-from-nested-flow-filters-strategy-impl.ts index d79070bb..031d63d3 100644 --- a/src/domain-services/flows/strategy/impl/get-flowIds-flow-from-nested-flow-filters-strategy-impl.ts +++ b/src/domain-services/flows/strategy/impl/get-flowIds-flow-from-nested-flow-filters-strategy-impl.ts @@ -87,16 +87,27 @@ export class GetFlowIdsFromNestedFlowFiltersStrategyImpl flowsLegacyId ); + if (flowIDsFromNestedFlowFilters.length === 0) { + return { flows: [] }; + } // Once gathered and disjoined the flowIDs from the nestedFlowFilters // Look after this uniqueFlows in the flow table // To verify the flow is not deleted - const getFlowArgs = { - databaseConnection, - orderBy: defaultFlowOrderBy(), - whereClauses: buildSearchFlowsConditions(flowIDsFromNestedFlowFilters), - }; - const uniqueFlowsNotDeleted = - await this.flowService.getFlowsAsUniqueFlowEntity(getFlowArgs); - return { flows: uniqueFlowsNotDeleted }; + const uniqueFlowEntitiesNotDeleted = []; + + // TEMP fix + for (let i = 0; i < flowIDsFromNestedFlowFilters.length; i += 1000) { + const getFlowArgs = { + databaseConnection, + orderBy: defaultFlowOrderBy(), + whereClauses: buildSearchFlowsConditions( + flowIDsFromNestedFlowFilters.slice(i, i + 1000) + ), + }; + const uniqueFlowsNotDeleted = + await this.flowService.getFlowsAsUniqueFlowEntity(getFlowArgs); + uniqueFlowEntitiesNotDeleted.push(...uniqueFlowsNotDeleted); + } + return { flows: uniqueFlowEntitiesNotDeleted }; } } diff --git a/src/domain-services/flows/strategy/impl/get-flowIds-flow-object-conditions-strategy-impl.ts b/src/domain-services/flows/strategy/impl/get-flowIds-flow-object-conditions-strategy-impl.ts index c1adc62c..1d2d4923 100644 --- a/src/domain-services/flows/strategy/impl/get-flowIds-flow-object-conditions-strategy-impl.ts +++ b/src/domain-services/flows/strategy/impl/get-flowIds-flow-object-conditions-strategy-impl.ts @@ -47,14 +47,21 @@ export class GetFlowIdsFromObjectConditionsStrategyImpl // 3. Once we have the flowIDs // Search in the Flow table for the flowIDs and versionIDs // To verify that the flows are not deleted + const uniqueFlowsNotDeleted: UniqueFlowEntity[] = []; + // TEMP fix + for (let i = 0; i < flowsFromFilteredFlowObjects.length; i += 1000) { + const getFlowArgs = { + databaseConnection, + orderBy: defaultFlowOrderBy(), + whereClauses: buildSearchFlowsConditions( + flowsFromFilteredFlowObjects.slice(i, i + 1000) + ), + }; + const uniqueFlowsNotDeletedSlice = + await this.flowService.getFlowsAsUniqueFlowEntity(getFlowArgs); + uniqueFlowsNotDeleted.push(...uniqueFlowsNotDeletedSlice); + } - const searchFlowArgs = { - databaseConnection, - orderBy: defaultFlowOrderBy(), - whereClauses: buildSearchFlowsConditions(flowsFromFilteredFlowObjects), - }; - const uniqueFlowsNotDeleted = - await this.flowService.getFlowsAsUniqueFlowEntity(searchFlowArgs); return { flows: uniqueFlowsNotDeleted }; } } diff --git a/src/domain-services/flows/strategy/impl/search-flow-by-filters-strategy-impl.ts b/src/domain-services/flows/strategy/impl/search-flow-by-filters-strategy-impl.ts index 01f431f2..c86cd0f4 100644 --- a/src/domain-services/flows/strategy/impl/search-flow-by-filters-strategy-impl.ts +++ b/src/domain-services/flows/strategy/impl/search-flow-by-filters-strategy-impl.ts @@ -18,6 +18,7 @@ import { defaultFlowOrderBy, intersectUniqueFlowEntities, mapFlowOrderBy, + mergeUniqueEntities, prepareFlowConditions, prepareFlowStatusConditions, } from './utils'; @@ -190,11 +191,12 @@ export class SearchFlowByFiltersStrategy implements FlowSearchStrategy { statusFilter ); + const orderByForFlow = isSortByEntity ? defaultFlowOrderBy() : orderBy; const flows: UniqueFlowEntity[] = await this.flowService.getFlowsAsUniqueFlowEntity({ databaseConnection, conditions: flowConditions, - orderBy, + orderBy: orderByForFlow, }); // If after this filter we have no flows, we can return an empty array @@ -227,12 +229,27 @@ export class SearchFlowByFiltersStrategy implements FlowSearchStrategy { // We are going to sort the deduplicated flows // using the sortByFlowIDs if there are any let sortedFlows: UniqueFlowEntity[] = []; - if (sortByFlowIDs.length > 0) { - // We need to keep only present in the deduplicatedFlows that way the sorted Flows get only the filtered ones but sorted + // There are 3 scenarios: + // 1. While sorting we have more flows 'sorted' than deduplicatedFlows + // That means we can apply the filterSorting pattern to gain a bit of performance + // 2. While sorting we have the same amount or less flows 'sorted' than deduplicatedFlows + // That means we need to keep the sortedFilters and then keep the rest of deduplicatedFlows thar are not in sortedFlows + // If we don't do this it may cause that just changing the orderBy we get different results + // Because we get rid of those flows that are not present in the sortedFlows list + // 3. There are no sortByFlowIDs + // That means we need to keep all the deduplicatedFlows as they come + if (sortByFlowIDs.length > deduplicatedFlows.length) { + sortedFlows = intersectUniqueFlowEntities( + sortByFlowIDs, + deduplicatedFlows + ); + } else if (sortByFlowIDs.length <= deduplicatedFlows.length) { sortedFlows = intersectUniqueFlowEntities( sortByFlowIDs, deduplicatedFlows ); + + sortedFlows = mergeUniqueEntities(sortedFlows, deduplicatedFlows); } else { sortedFlows = deduplicatedFlows; } diff --git a/src/domain-services/flows/strategy/impl/utils.ts b/src/domain-services/flows/strategy/impl/utils.ts index 874775d2..a4459612 100644 --- a/src/domain-services/flows/strategy/impl/utils.ts +++ b/src/domain-services/flows/strategy/impl/utils.ts @@ -62,7 +62,7 @@ export function mapFlowCategoryConditionsToWhereClause( return whereClause; } - return {}; + return null; } export function mergeFlowIDsFromFilteredFlowObjectsAndFlowCategories( @@ -174,16 +174,27 @@ export function mergeUniqueEntities( listA: UniqueFlowEntity[], listB: UniqueFlowEntity[] ): UniqueFlowEntity[] { - const entityMap = new Map(); + if (listA.length === 0) { + return listB; + } - for (const entity of [...listA, ...listB]) { - const key = `${entity.id}_${entity.versionID}`; - if (!entityMap.has(key)) { - entityMap.set(key, entity); + if (listB.length === 0) { + return listA; + } + + // Convert the lists into a set for efficient lookup + const entityMapListA = new Set(listA.map(mapUniqueFlowEntitisSetKeyToSetkey)); + + const entityMapListB = new Set(listB.map(mapUniqueFlowEntitisSetKeyToSetkey)); + + for (const key of entityMapListB) { + if (!entityMapListA.has(key)) { + entityMapListA.add(key); } } - return [...entityMap.values()]; + // Convert the keys back to UniqueFlowEntity objects + return mapUniqueFlowEntitisSetKeyToUniqueFlowEntity(entityMapListA); } export function intersectUniqueFlowEntities( @@ -200,16 +211,14 @@ export function intersectUniqueFlowEntities( return lists[0]; } - // Helper function to create a string key for comparison - const createKey = (entity: UniqueFlowEntity) => - `${entity.id}_${entity.versionID}`; - // Convert the first list into a set for efficient lookup - const initialSet = new Set(lists[0].map(createKey)); + const initialSet = new Set(lists[0].map(mapUniqueFlowEntitisSetKeyToSetkey)); // Intersect the remaining lists with the initial set for (let i = 1; i < lists.length; i++) { - const currentSet = new Set(lists[i].map(createKey)); + const currentSet = new Set( + lists[i].map(mapUniqueFlowEntitisSetKeyToSetkey) + ); for (const key of initialSet) { if (!currentSet.has(key)) { initialSet.delete(key); @@ -218,7 +227,19 @@ export function intersectUniqueFlowEntities( } // Convert the keys back to UniqueFlowEntity objects - return [...initialSet].map((key) => { + return mapUniqueFlowEntitisSetKeyToUniqueFlowEntity(initialSet); +} + +export function mapUniqueFlowEntitisSetKeyToSetkey( + entity: UniqueFlowEntity +): string { + return `${entity.id}_${entity.versionID}`; +} + +export function mapUniqueFlowEntitisSetKeyToUniqueFlowEntity( + set: Set +): UniqueFlowEntity[] { + return [...set].map((key) => { const [id, versionID] = key.split('_').map(Number); return { id, versionID } as UniqueFlowEntity; }); @@ -287,11 +308,7 @@ export function buildSearchFlowsConditions( flowFilters?: SearchFlowsFilters ): any { const whereClauses = uniqueFlowEntities.map((flow) => ({ - [Cond.AND]: [ - { id: flow.id }, - { versionID: flow.versionID }, - { deletedAt: null }, - ], + [Cond.AND]: [{ id: flow.id }, { versionID: flow.versionID }], })); if (flowFilters) {