From 2782b3208b1bc0bf4c4e817102a5b92c8996213d Mon Sep 17 00:00:00 2001 From: manelcecs Date: Wed, 24 Apr 2024 12:31:55 +0200 Subject: [PATCH] Add 'only-flow-conditions-strategy' implementation Add also utils to operate over flows --- .../flow-object/flow-object-service.ts | 13 +- src/domain-services/flows/flow-service.ts | 89 +++- src/domain-services/flows/model.ts | 7 +- .../only-flow-conditions-strategy-impl.ts | 65 +++ .../flows/strategy/impl/utils.ts | 412 ++++++++++++++++++ 5 files changed, 556 insertions(+), 30 deletions(-) create mode 100644 src/domain-services/flows/strategy/impl/only-flow-conditions-strategy-impl.ts create mode 100644 src/domain-services/flows/strategy/impl/utils.ts diff --git a/src/domain-services/flow-object/flow-object-service.ts b/src/domain-services/flow-object/flow-object-service.ts index 83229429..da97b58e 100644 --- a/src/domain-services/flow-object/flow-object-service.ts +++ b/src/domain-services/flow-object/flow-object-service.ts @@ -4,10 +4,13 @@ import { Op, type Condition, } from '@unocha/hpc-api-core/src/db/util/conditions'; -import { type InstanceOfModel } from '@unocha/hpc-api-core/src/db/util/types'; +import { type OrderByCond } from '@unocha/hpc-api-core/src/db/util/raw-model'; +import type { + FieldsOfModel, + InstanceOfModel, +} from '@unocha/hpc-api-core/src/db/util/types'; import { createBrandedValue } from '@unocha/hpc-api-core/src/util/types'; import { Service } from 'typedi'; -import { type OrderBy } from '../../utils/database-types'; import { type UniqueFlowEntity } from '../flows/model'; import { buildSearchFlowsObjectConditions } from '../flows/strategy/impl/utils'; import { type FlowObjectFilterGrouped } from './model'; @@ -16,6 +19,8 @@ import { buildWhereConditionsForFlowObjectFilters } from './utils'; // Local types definition to increase readability type FlowObjectModel = Database['flowObject']; type FlowObjectInstance = InstanceOfModel; +export type FlowObjectsFieldsDefinition = FieldsOfModel; +export type FlowObjectOrderByCond = OrderByCond; export type FlowObjectWhere = Condition; @Service() export class FlowObjectService { @@ -75,7 +80,7 @@ export class FlowObjectService { async getFlowsObjectsByFlows( models: Database, whereClauses: FlowObjectWhere, - orderBy?: OrderBy + orderBy?: FlowObjectOrderByCond ): Promise { const distinctColumns: Array = [ 'flowID', @@ -104,7 +109,7 @@ export class FlowObjectService { stopOnBatchSize: boolean, responseList: FlowObjectInstance[], flowObjectsWhere: FlowObjectWhere, - orderBy?: OrderBy + orderBy?: FlowObjectOrderByCond ): Promise { const reducedFlows = referenceList.slice(offset, offset + batchSize); diff --git a/src/domain-services/flows/flow-service.ts b/src/domain-services/flows/flow-service.ts index a552cf95..2aaa3cd3 100644 --- a/src/domain-services/flows/flow-service.ts +++ b/src/domain-services/flows/flow-service.ts @@ -20,10 +20,7 @@ import type { IGetFlowsArgs, UniqueFlowEntity, } from './model'; -import { - buildSearchFlowsConditions, - mapOrderByToEntityOrderBy, -} from './strategy/impl/utils'; +import { buildSearchFlowsConditions } from './strategy/impl/utils'; @Service() export class FlowService { @@ -56,12 +53,11 @@ export class FlowService { ): Promise { const entity = orderBy.subEntity ?? orderBy.entity; // Get the entity list - const mappedOrderBy = mapOrderByToEntityOrderBy(orderBy); // 'externalReference' is a special case // because it does have a direct relation with flow // and no direction if (entity === 'externalReference') { - const column = mappedOrderBy.column as keyof InstanceOfModel< + const column = orderBy.column as keyof InstanceOfModel< Database['externalReference'] >; const externalReferences = await database.externalReference.find({ @@ -90,9 +86,14 @@ export class FlowService { switch (entity) { case 'emergency': { // Get emergency entities sorted + const column = orderBy.column as keyof InstanceOfModel< + Database['emergency'] + >; + const orderByEmergency = { column, order: orderBy.order }; + const emergencies = await database.emergency.find({ - distinct: [mappedOrderBy.column, 'id'], - orderBy: mappedOrderBy, + distinct: [column, 'id'], + orderBy: orderByEmergency, }); entityIDsSorted = emergencies.map((emergency) => @@ -102,9 +103,14 @@ export class FlowService { } case 'globalCluster': { // Get globalCluster entities sorted + const column = orderBy.column as keyof InstanceOfModel< + Database['globalCluster'] + >; + const orderByGlobalCluster = { column, order: orderBy.order }; + const globalClusters = await database.globalCluster.find({ - distinct: [mappedOrderBy.column, 'id'], - orderBy: mappedOrderBy, + distinct: [column, 'id'], + orderBy: orderByGlobalCluster, }); entityIDsSorted = globalClusters.map((globalCluster) => @@ -114,9 +120,14 @@ export class FlowService { } case 'governingEntity': { // Get governingEntity entities sorted + const column = orderBy.column as keyof InstanceOfModel< + Database['governingEntity'] + >; + const orderByGoverningEntity = { column, order: orderBy.order }; + const governingEntities = await database.governingEntity.find({ - distinct: [mappedOrderBy.column, 'id'], - orderBy: mappedOrderBy, + distinct: [column, 'id'], + orderBy: orderByGoverningEntity, }); entityIDsSorted = governingEntities.map((governingEntity) => @@ -126,9 +137,14 @@ export class FlowService { } case 'location': { // Get location entities sorted + const column = orderBy.column as keyof InstanceOfModel< + Database['location'] + >; + const orderByLocation = { column, order: orderBy.order }; + const locations = await database.location.find({ - distinct: [mappedOrderBy.column, 'id'], - orderBy: mappedOrderBy, + distinct: [column, 'id'], + orderBy: orderByLocation, }); entityIDsSorted = locations.map((location) => location.id.valueOf()); @@ -136,9 +152,14 @@ export class FlowService { } case 'organization': { // Get organization entities sorted + const column = orderBy.column as keyof InstanceOfModel< + Database['organization'] + >; + const orderByOrganization = { column, order: orderBy.order }; + const organizations = await database.organization.find({ - distinct: [mappedOrderBy.column, 'id'], - orderBy: mappedOrderBy, + distinct: [column, 'id'], + orderBy: orderByOrganization, }); entityIDsSorted = organizations.map((organization) => @@ -148,9 +169,14 @@ export class FlowService { } case 'plan': { // Get plan entities sorted + const column = orderBy.column as keyof InstanceOfModel< + Database['plan'] + >; + const orderByPlan = { column, order: orderBy.order }; + const plans = await database.plan.find({ - distinct: [mappedOrderBy.column, 'id'], - orderBy: mappedOrderBy, + distinct: [column, 'id'], + orderBy: orderByPlan, }); entityIDsSorted = plans.map((plan) => plan.id.valueOf()); @@ -158,9 +184,14 @@ export class FlowService { } case 'project': { // Get project entities sorted + const column = orderBy.column as keyof InstanceOfModel< + Database['project'] + >; + const orderByProject = { column, order: orderBy.order }; + const projects = await database.project.find({ - distinct: [mappedOrderBy.column, 'id'], - orderBy: mappedOrderBy, + distinct: [column, 'id'], + orderBy: orderByProject, }); entityIDsSorted = projects.map((project) => project.id.valueOf()); @@ -168,9 +199,14 @@ export class FlowService { } case 'usageYear': { // Get usageYear entities sorted + const column = orderBy.column as keyof InstanceOfModel< + Database['usageYear'] + >; + const orderByUsageYear = { column, order: orderBy.order }; + const usageYears = await database.usageYear.find({ - distinct: [mappedOrderBy.column, 'id'], - orderBy: mappedOrderBy, + distinct: [column, 'id'], + orderBy: orderByUsageYear, }); entityIDsSorted = usageYears.map((usageYear) => usageYear.id.valueOf()); @@ -183,9 +219,14 @@ export class FlowService { entity.split(/[A-Z]/)[0] }Id` as keyof InstanceOfModel; + const column = orderBy.column as keyof InstanceOfModel< + Database['planVersion'] + >; + const orderByPlanVersion = { column, order: orderBy.order }; + const planVersions = await database.planVersion.find({ - distinct: [mappedOrderBy.column, entityKey], - orderBy: mappedOrderBy, + distinct: [column, entityKey], + orderBy: orderByPlanVersion, }); entityIDsSorted = planVersions.map((planVersion) => diff --git a/src/domain-services/flows/model.ts b/src/domain-services/flows/model.ts index 4a6850e7..1944ca10 100644 --- a/src/domain-services/flows/model.ts +++ b/src/domain-services/flows/model.ts @@ -2,14 +2,17 @@ import { type Database } from '@unocha/hpc-api-core/src/db'; import { type FlowId } from '@unocha/hpc-api-core/src/db/models/flow'; import { type Condition } from '@unocha/hpc-api-core/src/db/util/conditions'; import { type OrderByCond } from '@unocha/hpc-api-core/src/db/util/raw-model'; -import type { FieldsOfModel, InstanceOfModel } from '@unocha/hpc-api-core/src/db/util/types'; +import type { + FieldsOfModel, + InstanceOfModel, +} from '@unocha/hpc-api-core/src/db/util/types'; import { type SortOrder } from '../../utils/graphql/pagination'; import { type EntityDirection } from '../base-types'; export type FlowModel = Database['flow']; export type FlowInstance = InstanceOfModel; export type FlowWhere = Condition; -export type FlowFieldsDefinition = FieldsOfModel +export type FlowFieldsDefinition = FieldsOfModel; export type FlowOrderByCond = OrderByCond; // Can this be simplified somehow? export type UniqueFlowEntity = { id: FlowId; diff --git a/src/domain-services/flows/strategy/impl/only-flow-conditions-strategy-impl.ts b/src/domain-services/flows/strategy/impl/only-flow-conditions-strategy-impl.ts new file mode 100644 index 00000000..c46c3a44 --- /dev/null +++ b/src/domain-services/flows/strategy/impl/only-flow-conditions-strategy-impl.ts @@ -0,0 +1,65 @@ +import { Cond, Op } from '@unocha/hpc-api-core/src/db/util/conditions'; +import { Service } from 'typedi'; +import { FlowService } from '../../flow-service'; +import { type FlowWhere } from '../../model'; +import { + type FlowSearchArgs, + type FlowSearchStrategy, + type FlowSearchStrategyResponse, +} from '../flow-search-strategy'; +import { + mapFlowOrderBy, + prepareFlowConditions, + prepareFlowStatusConditions, +} from './utils'; + +@Service() +export class OnlyFlowFiltersStrategy implements FlowSearchStrategy { + constructor(private readonly flowService: FlowService) {} + + async search(args: FlowSearchArgs): Promise { + const { models, flowFilters, orderBy, limit, offset, statusFilter } = args; + // Map flowConditions to where clause + let flowConditions: FlowWhere = prepareFlowConditions(flowFilters); + + // Add status filter conditions if provided + flowConditions = prepareFlowStatusConditions(flowConditions, statusFilter); + + // Build conditions object + // We need to add the condition to filter the deletedAt field + const whereClause: FlowWhere = { + [Cond.AND]: [ + { + deletedAt: { + [Op.IS_NULL]: true, + }, + }, + flowConditions ?? {}, + ], + }; + + const orderByFlow = mapFlowOrderBy(orderBy); + + const [flows, countRes] = await Promise.all([ + this.flowService.getFlows({ + models, + conditions: whereClause, + offset, + orderBy: orderByFlow, + limit, + }), + await models.flow.count({ + where: whereClause, + }), + ]); + + // Map count result query to count object + const countObject = countRes; + + // on certain conditions, this conversion from 'bigint' to 'number' can cause a loss of precision + // But in order to reach that point, the number of flows would have to be in the billions + // that is not a realistic scenario for this application + // Nonetheless, we can validate that using Number.MAX_SAFE_INTEGER as a threshold + return { flows, count: Number(countObject) }; + } +} diff --git a/src/domain-services/flows/strategy/impl/utils.ts b/src/domain-services/flows/strategy/impl/utils.ts new file mode 100644 index 00000000..2322720f --- /dev/null +++ b/src/domain-services/flows/strategy/impl/utils.ts @@ -0,0 +1,412 @@ +import { type Database } from '@unocha/hpc-api-core/src/db'; +import { Cond, Op } from '@unocha/hpc-api-core/src/db/util/conditions'; +import type { InstanceDataOf } from '@unocha/hpc-api-core/src/db/util/model-definition'; +import { type InstanceOfModel } from '@unocha/hpc-api-core/src/db/util/types'; +import { createBrandedValue } from '@unocha/hpc-api-core/src/util/types'; +import { type OrderBy } from '../../../../utils/database-types'; +import { type SortOrder } from '../../../../utils/graphql/pagination'; +import { type EntityDirection } from '../../../base-types'; +import { type FlowObjectWhere } from '../../../flow-object/flow-object-service'; +import type { + FlowObjectFilterGrouped, + FlowObjectType, +} from '../../../flow-object/model'; +import type { + FlowCategory, + FlowObjectFilters, + SearchFlowsFilters, +} from '../../graphql/args'; +import type { FlowSortField, FlowStatusFilter } from '../../graphql/types'; +import type { + FlowFieldsDefinition, + FlowInstance, + FlowOrderByCond, + FlowOrderByWithSubEntity, + FlowWhere, + UniqueFlowEntity, +} from '../../model'; + +export const sortingColumnMapping: Map = new Map< + string, + string +>([ + ['reporterRefCode', 'refCode'], + ['sourceID', 'sourceID'], +]); + +export const defaultSearchFlowFilter: FlowWhere = { + deletedAt: null, +}; + +type FlowOrderByCommon = { + order: SortOrder; + direction?: EntityDirection; +}; + +export type FlowOrderBy = FlowOrderByCommon & + ( + | { + column: keyof FlowInstance; + entity: 'flow'; + } + | { + column: keyof InstanceOfModel; + entity: 'externalReference'; + } + | { + column: keyof InstanceOfModel; + entity: 'emergency'; + } + | { + entity: 'globalCluster'; + column: keyof InstanceOfModel; + } + | { + entity: 'governingEntity'; + column: keyof InstanceOfModel; + } + | { + entity: 'location'; + column: keyof InstanceOfModel; + } + | { + entity: 'organization'; + column: keyof InstanceOfModel; + } + | { + entity: 'plan'; + column: keyof InstanceOfModel; + } + | { + entity: 'usageYear'; + column: keyof InstanceOfModel; + } + | { + entity: 'planVersion'; + column: keyof InstanceOfModel; + } + | { + entity: 'project'; + column: keyof InstanceOfModel; + } + ); + +export const mapFlowCategoryConditionsToWhereClause = ( + flowCategoryConditions: FlowCategory[] +) => { + if (flowCategoryConditions.length > 0) { + let whereClause = {}; + // Map category filters + // getting Id when possible + // or name and group otherwise + const categoryIdFilters: number[] = []; + const categoryFilters = new Map(); + for (const categoryFilter of flowCategoryConditions) { + if (categoryFilter.id) { + categoryIdFilters.push(categoryFilter.id); + } else if (categoryFilter.group && categoryFilter.name) { + const group = categoryFilter.group; + const name = categoryFilter.name; + + const groupsNamesFilter = categoryFilters.get(group) ?? []; + + groupsNamesFilter.push(name); + categoryFilters.set(group, groupsNamesFilter); + } + } + + if (categoryIdFilters.length > 0) { + whereClause = { + ...whereClause, + id: { + [Op.IN]: categoryIdFilters, + }, + }; + } + + // For each entry of the group name + // add a condition to the where clause + // with the names associated to the group + // both in the same AND clause + for (const [group, names] of categoryFilters) { + whereClause = { + ...whereClause, + [Cond.AND]: [ + { + group: { + [Op.LIKE]: group, + }, + name: { + [Op.IN]: names, + }, + }, + ], + }; + } + return whereClause; + } + + return null; +}; + +export const mapFlowOrderBy = ( + orderBy?: FlowOrderByWithSubEntity +): OrderBy => { + if (!orderBy || orderBy.entity !== 'flow') { + return defaultFlowOrderBy(); + } + + return { + column: orderBy.column as keyof InstanceDataOf, + order: orderBy.order, + }; +}; + +export const defaultFlowOrderBy = (): FlowOrderByCond => { + return { + column: 'updatedAt', + order: 'desc', + } satisfies FlowOrderByCond; +}; + +export const prepareFlowConditions = ( + flowFilters: SearchFlowsFilters +): FlowWhere => { + let flowConditions: FlowWhere = { ...defaultSearchFlowFilter }; + + if (flowFilters) { + for (const [key, value] of Object.entries(flowFilters)) { + if (value !== undefined) { + if (key === 'id') { + const brandedIDs = value.map((id: number) => createBrandedValue(id)); + flowConditions[key] = { [Op.IN]: brandedIDs }; + } else { + const typedKey = key as keyof FlowWhere; + flowConditions = { ...flowConditions, [typedKey]: value }; + } + } + } + } + + return flowConditions satisfies FlowWhere; +}; + +export const mergeUniqueEntities = ( + listA: UniqueFlowEntity[], + listB: UniqueFlowEntity[] +): UniqueFlowEntity[] => { + if (listA.length === 0) { + return listB; + } + + if (listB.length === 0) { + return listA; + } + + // Convert the lists into a set for efficient lookup + const entityMapListA = new Set(listA.map(mapUniqueFlowEntitisSetKeyToSetkey)); + + const entityMapListB = new Set(listB.map(mapUniqueFlowEntitisSetKeyToSetkey)); + + for (const key of entityMapListB) { + if (!entityMapListA.has(key)) { + entityMapListA.add(key); + } + } + + // Convert the keys back to UniqueFlowEntity objects + return mapUniqueFlowEntitisSetKeyToUniqueFlowEntity(entityMapListA); +}; + +export const intersectUniqueFlowEntities = ( + ...lists: UniqueFlowEntity[][] +): UniqueFlowEntity[] => { + // If any of the lists is empty, remove it + lists = lists.filter((list) => list.length > 0); + + if (lists.length === 0) { + return []; + } + + if (lists.length === 1) { + return lists[0]; + } + + // Convert the first list into a set for efficient lookup + const initialSet = new Set(lists[0].map(mapUniqueFlowEntitisSetKeyToSetkey)); + + // Intersect the remaining lists with the initial set + for (let i = 1; i < lists.length; i++) { + const currentSet = new Set( + lists[i].map(mapUniqueFlowEntitisSetKeyToSetkey) + ); + for (const key of initialSet) { + if (!currentSet.has(key)) { + initialSet.delete(key); + } + } + } + + // Convert the keys back to UniqueFlowEntity objects + return mapUniqueFlowEntitisSetKeyToUniqueFlowEntity(initialSet); +}; + +export const mapUniqueFlowEntitisSetKeyToSetkey = ( + entity: UniqueFlowEntity +): string => { + return `${entity.id}_${entity.versionID}`; +}; + +export const mapUniqueFlowEntitisSetKeyToUniqueFlowEntity = ( + set: Set +): UniqueFlowEntity[] => { + return [...set].map((key) => { + const [id, versionID] = key.split('_').map(Number); + return { id: createBrandedValue(id), versionID } satisfies UniqueFlowEntity; + }); +}; + +export const prepareFlowStatusConditions = ( + flowConditions: FlowWhere, + statusFilter: FlowStatusFilter | null +): FlowWhere => { + if (statusFilter) { + if (statusFilter === 'new') { + // Flows with version 1 are considered new + flowConditions = { ...flowConditions, versionID: 1 }; + } else if (statusFilter === 'updated') { + // Flows with version greater than 1 are considered updated + flowConditions = { ...flowConditions, versionID: { [Op.GT]: 1 } }; + } + } + return flowConditions; +}; + +export const buildSearchFlowsConditions = ( + uniqueFlowEntities: UniqueFlowEntity[], + flowFilters?: FlowWhere +): FlowWhere => { + const whereClauses = uniqueFlowEntities.map((flow) => ({ + [Cond.AND]: [ + { id: flow.id }, + flow.versionID ? { versionID: flow.versionID } : {}, + ], + })); + + if (flowFilters) { + return { + [Cond.AND]: [ + { deletedAt: null }, + { [Cond.OR]: whereClauses }, + flowFilters, + ], + } satisfies FlowWhere; + } + + return { + [Cond.OR]: whereClauses, + }; +}; + +export const buildSearchFlowsObjectConditions = ( + uniqueFlowEntities: UniqueFlowEntity[], + flowObjectsWhere?: FlowObjectWhere +): FlowObjectWhere => { + const whereClauses = uniqueFlowEntities.map((flow) => ({ + [Cond.AND]: [ + { flowID: flow.id }, + flow.versionID ? { versionID: flow.versionID } : {}, + ], + })); + + if (flowObjectsWhere && Object.entries(flowObjectsWhere).length > 0) { + return { + [Cond.AND]: [flowObjectsWhere, ...whereClauses], + } satisfies FlowObjectWhere; + } + + return { + [Cond.OR]: whereClauses, + }; +}; + +export const mapFlowFiltersToFlowObjectFiltersGrouped = ( + flowObjectFilters: FlowObjectFilters[] +): FlowObjectFilterGrouped => { + const flowObjectFilterGrouped = new Map< + FlowObjectType, + Map + >(); + + for (const flowObjectFilter of flowObjectFilters) { + const objectType = flowObjectFilter.objectType; + const flowDirection = flowObjectFilter.direction; + const objectId = flowObjectFilter.objectID; + + // Get the map of flow object IDs for the given object type + // Or create a new map if it doesn't exist + const directionWithIDsMap = + flowObjectFilterGrouped.get(objectType) ?? + new Map(); + + // Get the list of flow object IDs for the given direction + // Or create a new list if it doesn't exist + const flowObjectIDs = directionWithIDsMap.get(flowDirection) ?? []; + flowObjectIDs.push(objectId); + + // Update the map with the new list of flow object IDs for the given direction + directionWithIDsMap.set(flowDirection, flowObjectIDs); + + // Update the map with the new map of direction+ids for the given object type + flowObjectFilterGrouped.set(objectType, directionWithIDsMap); + } + + return flowObjectFilterGrouped; +}; + +export const buildOrderBy = ( + sortField?: FlowSortField | string, + sortOrder?: SortOrder +): FlowOrderByWithSubEntity => { + const orderBy: FlowOrderByWithSubEntity = { + column: sortField ?? 'updatedAt', + order: sortOrder ?? ('desc' as SortOrder), + direction: undefined, + entity: 'flow', + }; + + // Check if sortField is a nested property + if (orderBy.column.includes('.')) { + // OrderBy can came in the format: + // column: 'organizations.source.name' + // or in the format: + // column: 'flow.updatedAt' + // or in the format: + // column: 'planVersion.source.name' + // in this last case, we need to look after the capitalized letter + // that will indicate the entity + // and the whole word will be the subEntity + const struct = orderBy.column.split('.'); + + if (struct.length === 2) { + orderBy.column = struct[1]; + orderBy.entity = struct[0]; + } else if (struct.length === 3) { + orderBy.column = struct[2]; + orderBy.direction = struct[1] as EntityDirection; + + // We need to look after the '-' character + // [0] will indicate the entity + // and [1] will be the subEntity + const splitted = struct[0].split('-'); + const entity = splitted[0]; + orderBy.entity = entity; + + if (entity === struct[0]) { + orderBy.subEntity = struct[0]; + } + } + } + + return orderBy; +};