From fea6179e0f634ca25eb7616ffc70f73e2dedd536 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Santos?= Date: Wed, 18 Oct 2023 19:36:14 +0100 Subject: [PATCH] Path model fixed --- src/models/Path.ts | 62 ++++++++++++++++++++++++------------------- src/models/Process.ts | 12 ++++----- src/models/Triple.ts | 6 +++++ 3 files changed, 47 insertions(+), 33 deletions(-) diff --git a/src/models/Path.ts b/src/models/Path.ts index 5aa553e..b6b67a1 100644 --- a/src/models/Path.ts +++ b/src/models/Path.ts @@ -1,12 +1,13 @@ import { Types, Document } from 'mongoose'; -import { UrlType } from '@derzis/common'; +import { urlValidator } from '@derzis/common'; import { prop, index, pre, getModelForClass } from '@typegoose/typegoose'; import { TripleClass, - ProcessClass, Triple, Process, ProcessTriple, + ProcessClass, + TripleDocument, } from '@derzis/models'; @pre('save', function () { @@ -32,12 +33,12 @@ class ResourceCount { @prop({ default: 0 }) public count!: number; - @prop({ default: [] }) - public elems!: UrlType[]; + @prop({ default: [], validate: urlValidator }) + public elems!: string[]; } class HeadClass { - @prop({ required: true }) - public url!: UrlType; + @prop({ required: true, validate: urlValidator }) + public url!: string; @prop({ required: true }) public domain!: string; @@ -51,12 +52,22 @@ type RecursivePartial = { : T[P]; }; +type PathSkeleton = Pick & + RecursivePartial & { + predicates: Pick; + nodes: Pick; + }; + class PathClass { + _id!: Types.ObjectId; + createdAt!: Date; + updatedAt!: Date; + @prop({ required: true }) public processId!: string; - @prop({ required: true }) - public seed!: UrlType; + @prop({ required: true, validate: urlValidator }) + public seed!: string; @prop({ required: true }) public head!: HeadClass; @@ -64,8 +75,8 @@ class PathClass { @prop({ default: [] }) public predicates!: ResourceCount; - @prop() - public lastPredicate?: UrlType; + @prop({ validate: urlValidator }) + public lastPredicate?: string; @prop({ default: [] }) public nodes!: ResourceCount; @@ -102,11 +113,8 @@ class PathClass { return true; } - public tripleIsOutOfBounds( - t: TripleClass, - process: ProcessDocumentClass - ): boolean { - const pathPreds: Set = new Set(this.predicates.elems); + public tripleIsOutOfBounds(t: TripleClass, process: ProcessClass): boolean { + const pathPreds: Set = new Set(this.predicates.elems); return ( this.nodes.count >= process.params.maxPathLength || (!pathPreds.has(t.predicate) && @@ -114,9 +122,9 @@ class PathClass { ); } - public extendWithExistingTriples(): Promise<{ - newPaths: PathClass[]; - procTriples: string[]; + public async extendWithExistingTriples(): Promise<{ + newPaths: PathSkeleton[]; + procTriples: Types.ObjectId[]; }> { // if path has outOfBounds triple, try to extend with that if (!!this.outOfBounds) { @@ -124,7 +132,7 @@ class PathClass { const process = await Process.findOne({ pid: this.processId }); if ( t && - !this.tripleIsOutOfBounds(t, process) && + !this.tripleIsOutOfBounds(t, process!) && process?.whiteBlackListsAllow(t!) ) { const newHeadUrl: string = @@ -150,13 +158,13 @@ class PathClass { } } // find triples which include the head but dont belong to the path yet - let triples: TripleClass[] = await Triple.find({ + let triples: TripleDocument[] = await Triple.find({ nodes: { $eq: this.head.url, $nin: this.nodes.elems }, }); return this.extend(triples); } - public copy(): RecursivePartial { + public copy(): PathSkeleton { const copy = { processId: this.processId, seed: this.seed, @@ -167,10 +175,10 @@ class PathClass { return copy; } - public extend( - triples: TripleClass[] - ): Promise<{ newPaths: PathClass[]; procTriples: string[] }> { - let newPaths: { [prop: string]: { [newHead: string]: PathClass } } = {}; + public async extend( + triples: TripleDocument[] + ): Promise<{ newPaths: PathSkeleton[]; procTriples: Types.ObjectId[] }> { + let newPaths: { [prop: string]: { [newHead: string]: PathSkeleton } } = {}; let procTriples: Types.ObjectId[] = []; const process = await Process.findOne({ pid: this.processId }); @@ -187,7 +195,7 @@ class PathClass { const np = this.copy(); np.head.url = newHeadUrl; - if (this.tripleIsOutOfBounds(t, process)) { + if (this.tripleIsOutOfBounds(t, process!)) { np.outOfBounds = t._id; } else { procTriples.push(t._id); @@ -199,7 +207,7 @@ class PathClass { newPaths[prop][newHeadUrl] = np; } } - const nps: RecursivePartial[] = []; + const nps: PathSkeleton[] = []; Object.values(newPaths).forEach((x) => Object.values(x).forEach((y) => nps.push(y)) ); diff --git a/src/models/Process.ts b/src/models/Process.ts index ea188c3..11ce57b 100644 --- a/src/models/Process.ts +++ b/src/models/Process.ts @@ -1,6 +1,6 @@ import { Types, Document } from 'mongoose'; import { Resource } from './Resource'; -import { Triple, TripleClass } from './Triple'; +import { Triple, TripleClass, TripleDocument } from './Triple'; import { humanize } from 'humanize-digest'; import { Domain } from './Domain'; import { Path, PathDocument } from './Path'; @@ -38,11 +38,11 @@ class NotificationClass { public ssePath?: string; } class ParamsClass { - @prop({ default: 2 }) - public maxPathLength?: number; + @prop({ default: 2, required: true }) + public maxPathLength!: number; - @prop({ default: 1 }) - public maxPathProps?: number; + @prop({ default: 1, required: true }) + public maxPathProps!: number; @prop({ default: [] }) public whiteList?: string[]; @@ -162,7 +162,7 @@ class ProcessClass { } } - public async extendPaths(triplesByNode: { [url: string]: TripleClass[] }) { + public async extendPaths(triplesByNode: { [url: string]: TripleDocument[] }) { const newHeads = Object.keys(triplesByNode); const paths = await Path.find({ processId: this.pid, diff --git a/src/models/Triple.ts b/src/models/Triple.ts index 65ace15..41de8dd 100644 --- a/src/models/Triple.ts +++ b/src/models/Triple.ts @@ -8,6 +8,7 @@ import { index, getModelForClass, ReturnModelType, + mongoose, } from '@typegoose/typegoose'; import { Document } from 'cheerio'; @@ -23,6 +24,11 @@ import { Document } from 'cheerio'; @index({ nodes: 1 }) @index({ subject: 1, predicate: 1, object: 1 }, { unique: true }) class TripleClass { + _id!: mongoose.Types.ObjectId; + + createdAt!: Date; + updatedAt!: Date; + @prop({ required: true, validate: urlValidator }) public subject!: string;