From 64bcb4ba1c33055afe6a8f1ab8357a8078edee57 Mon Sep 17 00:00:00 2001 From: Rob Donigian <167127316+rdonigian@users.noreply.github.com> Date: Tue, 17 Dec 2024 18:30:21 -0500 Subject: [PATCH 1/2] Add filters & sorting for pnp extraction errors on Progress Reports (#3335) Co-authored-by: Carson Full --- .../extraction-result.dto.ts | 17 +++++++-- .../pnp-extraction-result.neo4j.repository.ts | 36 ++++++++++++++++++- .../dto/progress-report-list.dto.ts | 4 +++ ...extra-for-periodic-interface.repository.ts | 13 +++++++ .../progress-report.repository.ts | 10 ++++++ 5 files changed, 77 insertions(+), 3 deletions(-) diff --git a/src/components/pnp/extraction-result/extraction-result.dto.ts b/src/components/pnp/extraction-result/extraction-result.dto.ts index 203bd251fb..0de3ee3201 100644 --- a/src/components/pnp/extraction-result/extraction-result.dto.ts +++ b/src/components/pnp/extraction-result/extraction-result.dto.ts @@ -1,10 +1,11 @@ -import { Field, InterfaceType, ObjectType } from '@nestjs/graphql'; +import { Field, InputType, InterfaceType, ObjectType } from '@nestjs/graphql'; import { many, Many } from '@seedcompany/common'; import { stripIndent } from 'common-tags'; import { UUID } from 'node:crypto'; +import { keys as keysOf } from 'ts-transformer-keys'; import { Merge } from 'type-fest'; import * as uuid from 'uuid'; -import { EnumType, ID, IdField, makeEnum } from '~/common'; +import { EnumType, ID, IdField, makeEnum, SecuredProps } from '~/common'; import { InlineMarkdownScalar } from '~/common/markdown.scalar'; import { Cell } from '~/common/xlsx.util'; @@ -118,8 +119,20 @@ export type StoredProblem = Pick & { context: { [x: string]: unknown }; }; +@InputType() +export class PnpExtractionResultFilters { + @Field(() => Boolean, { + nullable: true, + description: 'Only extraction results containing errors', + }) + readonly hasError?: boolean; +} + @InterfaceType() export abstract class PnpExtractionResult { + static readonly Props = keysOf(); + static readonly SecuredProps = keysOf>(); + constructor(private readonly fileVersionId: ID<'FileVersion'>) {} readonly problems = new Map(); diff --git a/src/components/pnp/extraction-result/pnp-extraction-result.neo4j.repository.ts b/src/components/pnp/extraction-result/pnp-extraction-result.neo4j.repository.ts index ed905c75c0..ef2ccf78ac 100644 --- a/src/components/pnp/extraction-result/pnp-extraction-result.neo4j.repository.ts +++ b/src/components/pnp/extraction-result/pnp-extraction-result.neo4j.repository.ts @@ -4,10 +4,22 @@ import { inArray, node, relation } from 'cypher-query-builder'; import { SetNonNullable } from 'type-fest'; import { ID, PublicOf } from '~/common'; import { CommonRepository } from '~/core/database'; -import { apoc, collect, exp, merge, variable } from '~/core/database/query'; +import { + apoc, + collect, + count, + defineSorters, + exp, + filter, + merge, + SortCol, + variable, +} from '~/core/database/query'; import { PnpExtractionResult, + PnpExtractionResultFilters, PnpProblemType, + PnpProblemSeverity as Severity, StoredProblem, } from './extraction-result.dto'; import { PnpExtractionResultRepository } from './pnp-extraction-result.edgedb.repository'; @@ -117,3 +129,25 @@ export class PnpExtractionResultNeo4jRepository .run(); } } + +export const pnpExtractionResultFilters = filter.define( + () => PnpExtractionResultFilters, + { + hasError: filter.pathExists([ + node('node'), + relation('out', '', 'problem'), + node('', { severity: Severity.Error }), + ]), + }, +); + +export const pnpExtractionResultSorters = defineSorters(PnpExtractionResult, { + totalErrors: (query) => + query + .match([ + node('node'), + relation('out', 'problem', 'problem'), + node('type', { severity: Severity.Error }), + ]) + .return(count('problem').as('sortValue')), +}); diff --git a/src/components/progress-report/dto/progress-report-list.dto.ts b/src/components/progress-report/dto/progress-report-list.dto.ts index 14c224447e..0fee60a01b 100644 --- a/src/components/progress-report/dto/progress-report-list.dto.ts +++ b/src/components/progress-report/dto/progress-report-list.dto.ts @@ -8,6 +8,7 @@ import { } from '~/common'; import { EngagementFilters } from '../../engagement/dto'; import { PeriodicReportListInput } from '../../periodic-report/dto'; +import { PnpExtractionResultFilters } from '../../pnp/extraction-result'; import { ProgressSummaryFilters } from '../../progress-summary/dto'; import { ProgressReportStatus } from './progress-report-status.enum'; import { ProgressReport } from './progress-report.entity'; @@ -27,6 +28,9 @@ export abstract class ProgressReportFilters extends PickType( @FilterField(() => EngagementFilters) readonly engagement?: EngagementFilters & {}; + + @FilterField(() => PnpExtractionResultFilters) + readonly pnpExtractionResult?: PnpExtractionResultFilters & {}; } @InputType() diff --git a/src/components/progress-report/progress-report-extra-for-periodic-interface.repository.ts b/src/components/progress-report/progress-report-extra-for-periodic-interface.repository.ts index 396a7b9706..5bb598f976 100644 --- a/src/components/progress-report/progress-report-extra-for-periodic-interface.repository.ts +++ b/src/components/progress-report/progress-report-extra-for-periodic-interface.repository.ts @@ -16,6 +16,7 @@ import { } from '~/core/database/query'; import { engagementSorters } from '../engagement/engagement.repository'; import { MergePeriodicReports } from '../periodic-report/dto'; +import { pnpExtractionResultSorters } from '../pnp/extraction-result/pnp-extraction-result.neo4j.repository'; import { SummaryPeriod } from '../progress-summary/dto'; import { progressSummarySorters } from '../progress-summary/progress-summary.repository'; import { ProgressReport, ProgressReportStatus as Status } from './dto'; @@ -50,6 +51,18 @@ export const progressReportExtrasSorters: DefinedSorters< SortFieldOf > = defineSorters(ProgressReport, { // eslint-disable-next-line @typescript-eslint/naming-convention + 'pnpExtractionResult.*': (query, input) => + query + .with('node as report') + .match([ + node('report'), + relation('out', '', 'reportFileNode'), + node('file', 'File'), + relation('out', '', 'pnpExtractionResult'), + node('node', 'PnpExtractionResult'), + ]) + .apply(sortWith(pnpExtractionResultSorters, input)), + // eslint-disable-next-line @typescript-eslint/naming-convention 'engagement.*': (query, input) => query .with('node as report') diff --git a/src/components/progress-report/progress-report.repository.ts b/src/components/progress-report/progress-report.repository.ts index e1906b8343..9bf3cc587e 100644 --- a/src/components/progress-report/progress-report.repository.ts +++ b/src/components/progress-report/progress-report.repository.ts @@ -15,6 +15,7 @@ import { } from '~/core/database/query'; import { engagementFilters } from '../engagement/engagement.repository'; import { progressReportSorters } from '../periodic-report/periodic-report.repository'; +import { pnpExtractionResultFilters } from '../pnp/extraction-result/pnp-extraction-result.neo4j.repository'; import { SummaryPeriod } from '../progress-summary/dto'; import { progressSummaryFilters } from '../progress-summary/progress-summary.repository'; import { @@ -109,5 +110,14 @@ export const progressReportFilters = filter.define( node('node', 'Engagement'), ]), ), + pnpExtractionResult: filter.sub(() => pnpExtractionResultFilters)((sub) => + sub.match([ + node('outer'), + relation('out', '', 'reportFileNode'), + node('file', 'File'), + relation('out', '', 'pnpExtractionResult'), + node('node', 'PnpExtractionResult'), + ]), + ), }, ); From 72c7acc9c30606efd0afd64a304acbedf3f7019a Mon Sep 17 00:00:00 2001 From: Carson Full Date: Tue, 17 Dec 2024 17:30:29 -0600 Subject: [PATCH 2/2] Fix engagement/pnp date match to exact match days exactly (#3344) --- .../pnp/verifyEngagementDateRangeMatches.ts | 12 ++++-------- .../reextract-all-progress-reports.migration.ts | 2 +- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/src/components/pnp/verifyEngagementDateRangeMatches.ts b/src/components/pnp/verifyEngagementDateRangeMatches.ts index a10b075e4a..b9c31821d5 100644 --- a/src/components/pnp/verifyEngagementDateRangeMatches.ts +++ b/src/components/pnp/verifyEngagementDateRangeMatches.ts @@ -1,4 +1,5 @@ import { stripIndent } from 'common-tags'; +import { DateTime } from 'luxon'; import { DateInterval } from '~/common'; import { PnpPlanningExtractionResult, @@ -19,10 +20,7 @@ export function verifyEngagementDateRangeMatches( } const matches = - engagementRange && - pnpRange && - // PnP only specifies months; allow nuance on our side. - engagementRange.expandToFull('month').equals(pnpRange); + engagementRange && pnpRange && engagementRange.equals(pnpRange); if (matches) { return true; @@ -68,13 +66,11 @@ const MismatchedEngagementDateRange = PnpProblemType.register({ message: stripIndent` The PnP's **project dates** (\`${source}\`) are different from what is declared in the CORD Engagement/Project. - CORD: ${eng.toLocaleString({ month: 'long', year: 'numeric' })} - PnP: ${pnp.toLocaleString({ month: 'long', year: 'numeric' })} + CORD: ${eng.toLocaleString(DateTime.DATE_MED)} + PnP: ${pnp.toLocaleString(DateTime.DATE_MED)} Please adjust the dates in CORD or the PnP to match and be accurate. - CORD only requires the month & year to match the PnP since this is as granular as it gets. - If this is a cluster project and this engagement ("language") has started late or ends early, that difference can be declared in CORD's _Engagement dates_. `.replaceAll(/\n/g, ' \n'), // keep line breaks with MD's two space trailers diff --git a/src/components/progress-report/migrations/reextract-all-progress-reports.migration.ts b/src/components/progress-report/migrations/reextract-all-progress-reports.migration.ts index c1d6b4959f..bd383cf307 100644 --- a/src/components/progress-report/migrations/reextract-all-progress-reports.migration.ts +++ b/src/components/progress-report/migrations/reextract-all-progress-reports.migration.ts @@ -8,7 +8,7 @@ import { FileVersion } from '../../file/dto'; import { PeriodicReportUploadedEvent } from '../../periodic-report/events'; import { ProgressReport } from '../dto'; -@Migration('2024-12-16T16:33:00') +@Migration('2024-12-17T16:00:00') export class ReextractPnpProgressReportsMigration extends BaseMigration { constructor( private readonly eventBus: IEventBus,