Skip to content

Commit

Permalink
Temp: performance
Browse files Browse the repository at this point in the history
  • Loading branch information
@manelcecs
manelcecs committed Jan 10, 2024
1 parent 7d6ca42 commit 4d5719e
Show file tree
Hide file tree
Showing 5 changed files with 93 additions and 84 deletions.
27 changes: 26 additions & 1 deletion src/domain-services/flows/flow-service.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { Service } from 'typedi';
import { type FlowObjectType } from '../flow-object/model';
import { GetFlowsArgs, UniqueFlowEntity, type FlowOrderBy } from './model';
import {
applySearchFilters,
mapFlowOrderBy,
removeDuplicatesUniqueFlowEntities,
} from './strategy/impl/utils';
Expand All @@ -15,14 +16,38 @@ export class FlowService {
async getFlows(args: GetFlowsArgs) {
const { models, conditions, offset, orderBy, limit } = args;

return await models.flow.find({
return await models!.flow.find({
orderBy,
limit,
where: conditions,
offset,
});
}

async getFlowsAsUniqueFlowEntity(args: GetFlowsArgs): Promise<UniqueFlowEntity[]> {
const { databaseConnection, orderBy, conditions } = args;

let query = databaseConnection!.queryBuilder()
.distinct('id', 'versionID', orderBy.column) // Include orderBy.column in the distinct selection
.select('id', 'versionID')
.from('flow')
.whereNull('deletedAt')
.orderBy(orderBy.column, orderBy.order);

if(conditions) {
query = applySearchFilters(query, conditions);
}

const flows = await query;

const mapFlowsToUniqueFlowEntities = flows.map((flow) => ({
id: flow.id,
versionID: flow.versionID,
}));

return mapFlowsToUniqueFlowEntities;
}

async getFlowsCount(models: Database, conditions: any) {
return await models.flow.count({ where: conditions });
}
Expand Down
4 changes: 3 additions & 1 deletion src/domain-services/flows/model.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { type Database } from '@unocha/hpc-api-core/src/db';
import { type FlowId } from '@unocha/hpc-api-core/src/db/models/flow';
import { type InstanceDataOfModel } from '@unocha/hpc-api-core/src/db/util/raw-model';
import { type SortOrder } from '../../utils/graphql/pagination';
import Knex from 'knex';

export type FlowEntity = InstanceDataOfModel<Database['flow']>;

Expand All @@ -21,7 +22,8 @@ export type FlowOrderBy = {
export type FlowNestedDirection = 'source' | 'destination';

export type GetFlowsArgs = {
models: Database;
models?: Database;
databaseConnection?: Knex;
conditions: any;
offset?: number;
orderBy?: any;
Expand Down
60 changes: 0 additions & 60 deletions ...domain-services/flows/strategy/impl/get-flowIds-flow-category-conditions-strategy-impl.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,6 @@ export class GetFlowIdsFromCategoryConditionsStrategyImpl
}

const flows = await query;
console.log('flows', flows.length);
const mapFlows: UniqueFlowEntity[] = flows.map(
(flow) =>
({
Expand All @@ -114,65 +113,6 @@ export class GetFlowIdsFromCategoryConditionsStrategyImpl
);

return { flows: mapFlows };
// const whereClauseCategoryRef = {
// [Cond.OR]: [
// categoriesIds.length > 0
// ? {
// [Cond.AND]: [
// {
// categoryID: {
// [Op.IN]: categoriesIds,
// },
// },
// { objectType: 'flow' },
// ],
// }
// : {},
// categoriesIdsFromShortcutFilterIN.length > 0
// ? {
// [Cond.AND]: [
// {
// categoryID: {
// [Op.IN]: categoriesIdsFromShortcutFilterIN,
// },
// },
// { objectType: 'flow' },
// ],
// }
// : {},
// categoriesIdsFromShortcutFilterNOTIN.length > 0
// ? {
// [Cond.AND]: [
// {
// categoryID: {
// [Op.NOT_IN]: categoriesIdsFromShortcutFilterNOTIN,
// },
// },
// { objectType: 'flow' },
// ],
// }
// : {},
// ],
// };

// const categoryRefs = await this.categoryService.findCategoryRefs(
// models,
// whereClauseCategoryRef
// );

// Map category refs to flow IDs
// keep only unique values
// and return the list of flow IDs
// const mapFlowsToUniqueFlowEntities = categoryRefs.map((categoryRef) => {
// return {
// id: categoryRef.objectID,
// versionID: categoryRef.versionID,
// } as UniqueFlowEntity;
// })

// const flows = removeDuplicatesUniqueFlowEntities(mapFlowsToUniqueFlowEntities);

// return { flows };
}

generateWhereClause(
Expand Down
61 changes: 39 additions & 22 deletions src/domain-services/flows/strategy/impl/search-flow-by-filters-strategy-impl.ts
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
import { type FlowId } from '@unocha/hpc-api-core/src/db/models/flow';
import { Cond, Op } from '@unocha/hpc-api-core/src/db/util/conditions';
import { Cond } from '@unocha/hpc-api-core/src/db/util/conditions';
import { Service } from 'typedi';
import { FlowService } from '../../flow-service';
import { UniqueFlowEntity } from '../../model';
import { FlowEntity, UniqueFlowEntity } from '../../model';
import {
type FlowSearchArgs,
type FlowSearchStrategy,
Expand All @@ -13,10 +12,8 @@ import { GetFlowIdsFromCategoryConditionsStrategyImpl } from './get-flowIds-flow
import { GetFlowIdsFromObjectConditionsStrategyImpl } from './get-flowIds-flow-object-conditions-strategy-impl';
import {
intersectUniqueFlowEntities,
mapCountResultToCountObject,
mapFlowObjectConditions,
mapFlowOrderBy,
mergeUniqueEntities,
prepareFlowConditions,
sortEntitiesByReferenceList,
} from './utils';
Expand Down Expand Up @@ -63,30 +60,32 @@ export class SearchFlowByFiltersStrategy implements FlowSearchStrategy {
// In this case we fetch the list of flows from the database
// using the orderBy
// We can also filter by flowFilters
const flowConditions = prepareFlowConditions(flowFilters);
const orderByForFlow = mapFlowOrderBy(orderBy);

const flowsToSort = await this.flowService.getFlows({
models,
conditions: flowConditions,
orderBy: { column: orderBy.column, order: orderBy.order },
const flowsToSort: UniqueFlowEntity[] = await this.flowService.getFlowsAsUniqueFlowEntity({
databaseConnection,
conditions: flowFilters,
orderBy: orderByForFlow,
});

const flowIDsFromSortingEntity: UniqueFlowEntity[] = flowsToSort.map(
(flow) => ({ id: flow.id, versionID: flow.versionID })
);
// Since there can be many flowIDs returned
// This can cause 'Maximum call stack size exceeded' error
// When using the spread operator - a workaround is to use push fot each element
for (const flow of flowIDsFromSortingEntity) {
sortByFlowIDs.push(flow);
// also, we need to map the FlowEntity to UniqueFlowEntity
for (const flow of flowsToSort) {
const uniqueFlowEntity: UniqueFlowEntity = {
id: flow.id,
versionID: flow.versionID,
};
sortByFlowIDs.push(uniqueFlowEntity);
}
}

// Now we need to check if we need to filter by category
// if it's using any of the shorcuts
// or if there are any flowCategoryFilters
const isSearchByCategoryShotcut = shortcutFilter !== null;

const isSearchByCategoryShotcut = shortcutFilter !== null && shortcutFilter.length > 0;
const isFilterByCategory =
isSearchByCategoryShotcut || flowCategoryFilters?.length > 0;

Expand All @@ -101,6 +100,7 @@ export class SearchFlowByFiltersStrategy implements FlowSearchStrategy {
shortcutFilter,
flowObjectsConditions: undefined,
});

// Since there can be many flowIDs returned
// This can cause 'Maximum call stack size exceeded' error
// When using the spread operator - a workaround is to use push fot each element
Expand All @@ -123,7 +123,13 @@ export class SearchFlowByFiltersStrategy implements FlowSearchStrategy {
models,
flowObjectsConditions: flowObjectConditionsMap,
});
flowsFromObjectFilters.push(...flows);

// Since there can be many flowIDs returned
// This can cause 'Maximum call stack size exceeded' error
// When using the spread operator - a workaround is to use push fot each element
for (const flow of flows) {
flowsFromObjectFilters.push(flow);
}
}

// Lastly, we need to check if we need to filter by flow
Expand All @@ -134,13 +140,22 @@ export class SearchFlowByFiltersStrategy implements FlowSearchStrategy {
const flowsFromFlowFilters: UniqueFlowEntity[] = [];
if (isSortByEntity && isFilterByFlow) {
const flowConditions = prepareFlowConditions(flowFilters);
const flows = await this.flowService.getFlows({
const flows: FlowEntity[] = await this.flowService.getFlows({
models,
conditions: flowConditions,
orderBy: { column: orderBy.column, order: orderBy.order },
});

// Since there can be many flowIDs returned
// This can cause 'Maximum call stack size exceeded' error
// When using the spread operator - a workaround is to use push fot each element
// also, we need to map the FlowEntity to UniqueFlowEntity
for (const flow of flows) {
flowsFromFlowFilters.push({ id: flow.id, versionID: flow.versionID });
const uniqueFlowEntity: UniqueFlowEntity = {
id: flow.id,
versionID: flow.versionID,
};
sortByFlowIDs.push(uniqueFlowEntity);
}
}

Expand All @@ -165,6 +180,7 @@ export class SearchFlowByFiltersStrategy implements FlowSearchStrategy {
sortByFlowIDs
);


// Then we are going to slice the flows using the limit and offset
const reducedFlows: UniqueFlowEntity[] = sortedFlows.slice(
offset,
Expand All @@ -174,11 +190,12 @@ export class SearchFlowByFiltersStrategy implements FlowSearchStrategy {
// Once the list of elements is reduced, we need to build the conditions
const searchConditions = this.buildConditions(reducedFlows);

const orderByForFlow = mapFlowOrderBy(orderBy);

const flows = await this.flowService.getFlows({
models,
conditions: searchConditions,
limit,
orderBy: { column: orderBy.column, order: orderBy.order },
orderBy: orderByForFlow,
});

return { flows, count };
Expand Down
25 changes: 25 additions & 0 deletions src/domain-services/flows/strategy/impl/utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import {
type SearchFlowsFilters,
} from '../../graphql/args';
import { UniqueFlowEntity } from '../../model';
import Knex from 'knex';

/*
* Map structure:
Expand Down Expand Up @@ -304,3 +305,27 @@ export function removeDuplicatesUniqueFlowEntities(

return Array.from(uniqueEntities.values());
}

export function applySearchFilters(query: Knex.QueryBuilder, filters: SearchFlowsFilters): Knex.QueryBuilder {
// Check if 'id' filter is defined and apply it
if (filters.id !== null && filters.id !== undefined) {
query.whereIn('id', filters.id);
}

// Check if 'activeStatus' filter is defined and apply it
if (filters.activeStatus !== null && filters.activeStatus !== undefined) {
query.andWhere('activeStatus', filters.activeStatus);
}

// Check if 'amountUSD' filter is defined and apply it
if (filters.amountUSD !== null && filters.amountUSD !== undefined) {
query.andWhere('amountUSD', filters.amountUSD);
}

// Check if 'restricted' filter is defined and apply it
if (filters.restricted !== null && filters.restricted !== undefined) {
query.andWhere('restricted', filters.restricted);
}

return query;
}

0 comments on commit 4d5719e

Please sign in to comment.