From a7fa6f5ba01b1fe03cecc4aa9cce4ad9bd260259 Mon Sep 17 00:00:00 2001 From: Gerard Date: Mon, 5 Feb 2024 13:45:45 +0100 Subject: [PATCH] Fix importing ordered items --- CHANGELOG.md | 22 ++++++++- install/schema-sync/config.js | 2 + install/schema-sync/directus_config.js | 1 + src/collectionExporter.ts | 62 +++++++++++++++++++++++++- src/default_config.ts | 1 + src/types.ts | 1 + src/updateManager.ts | 2 +- 7 files changed, 87 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6701a94..43b88d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,24 @@ -# Version 1.6.2 +# Version 1.6.3 + + - Add `linkedFields` to fix inserting ordered items that might depend on each other. + - Specifically fixes importing of flow operations, update the directus_config and add `linkedFields`. + ```javascript + { + directus_operations: { + watch: ['operations'], + excludeFields: ['user_created'], + linkedFields: ['resolve', 'reject'], + query: { + filter: { + flow: { trigger: { _neq: 'webhook' } }, + }, + }, + }, + } + ``` + - Also fix auto IMPORT issue when mv_ts is null. + +## Version 1.6.2 - (postgres only) Fixed import issue due to default_value containing functions for auto increment diff --git a/install/schema-sync/config.js b/install/schema-sync/config.js index 3e24e5d..92ff59a 100644 --- a/install/schema-sync/config.js +++ b/install/schema-sync/config.js @@ -5,6 +5,7 @@ * collectionName: * watch: array of events to watch for changes, eg. 'posts.items', * excludeFields: (optional) array of fields to exclude from the export, + * m2oFields: (optional) array of fields to treat as many-to-one relationships for hierarchy, eg. ['parent'], * getKey: (optional) function to get the key for the item, defaults to primary key found on schema, * query: (optional) query to use when exporting the collection, valid options are: (limit=-1 | filter | sort) * prefix: (optional) prefix the exported json file with this string (useful for separating test data from production data) @@ -16,6 +17,7 @@ export const syncCustomCollections = { posts: { watch: ['posts.items'], excludeFields: ['user_created', 'user_updated'], + m2oFields: ['parent'], query: { filter: { shared: { _eq: true } diff --git a/install/schema-sync/directus_config.js b/install/schema-sync/directus_config.js index d04229d..bba2f2c 100644 --- a/install/schema-sync/directus_config.js +++ b/install/schema-sync/directus_config.js @@ -66,6 +66,7 @@ export const syncDirectusCollections = { directus_operations: { watch: ['operations'], excludeFields: ['user_created'], + linkedFields: ['resolve', 'reject'], query: { filter: { flow: { trigger: { _neq: 'webhook' } }, diff --git a/src/collectionExporter.ts b/src/collectionExporter.ts index 0fa7644..32438ba 100644 --- a/src/collectionExporter.ts +++ b/src/collectionExporter.ts @@ -64,7 +64,18 @@ class CollectionExporter implements IExporter { await writeFile(this.filePath, json); } + protected _settings: { + inclFields: Array; + exclFields: Array; + linkedFields: NonNullable + getKey: (o: Item) => PrimaryKey; + getPrimary: (o: Item) => PrimaryKey; + query: Query; + queryWithPrimary: Query; + } | null = null; protected async settings() { + if (this._settings) return this._settings; + const itemsSvc = await this._getService() const schema = itemsSvc.schema.collections[this.collection] @@ -95,9 +106,10 @@ class CollectionExporter implements IExporter { const queryWithPrimary: Query = exclFields.includes(schema.primary) ? { ...query, fields: [...inclFields, schema.primary] } : query; - return { + return this._settings = { inclFields, exclFields, + linkedFields: this.options.linkedFields || [], getKey, getPrimary, query, @@ -125,6 +137,46 @@ class CollectionExporter implements IExporter { return JSON.stringify(sortObject(items), null, 2); } + + /** + * Orders items so that items that are linked are inserted after the items they reference + * Only works with items that have a primary key + * Assumes items not in given items list are already in the database + * @param items + * @returns + */ + protected async sortbyIfLinked(items: Array) { + const { getPrimary, linkedFields } = await this.settings(); + if (!linkedFields.length) return false; + + const itemsMap = items.reduce((map, o) => { + o.__dependents = []; + map[getPrimary(o)] = o; + return map; + }, {} as Record); + + let hasDependents = false; + items.forEach(o => { + for (const fieldName of linkedFields) { + const value = o[fieldName]; + if (value && itemsMap[value]) { + hasDependents = true; + itemsMap[value].__dependents.push(o); + } + } + }); + + items.sort((a, b) => this.countDependents(b) - this.countDependents(a)); + items.forEach(o => delete o.__dependents); + + return true; + } + // Recursively count dependents + private countDependents(o: any): number { + if (!o.__dependents.length) return 0; + return o.__dependents.reduce((acc, o) => acc + this.countDependents(o), o.__dependents.length); + } + public async loadJSON(json: JSONString | null, merge = false) { if (!json) return null; const loadedItems = JSON.parse(json) as Array; @@ -182,7 +234,13 @@ class CollectionExporter implements IExporter { // Insert if (toInsert.length > 0) { this.logger.debug(`Inserting ${toInsert.length} x ${this.collection} items`); - await itemsSvc.createMany(toInsert); + if (await this.sortbyIfLinked(toInsert)) { + for (const item of toInsert) { + await itemsSvc.createOne(item); + } + } else { + await itemsSvc.createMany(toInsert); + } } // Update diff --git a/src/default_config.ts b/src/default_config.ts index 9ceaba0..c9d9ee4 100644 --- a/src/default_config.ts +++ b/src/default_config.ts @@ -70,6 +70,7 @@ export const syncDirectusCollections: ExportCollectionConfig = { directus_operations: { watch: ['operations'], excludeFields: ['user_created'], + linkedFields: ['resolve', 'reject'], }, directus_translations: { watch: ['translations'], diff --git a/src/types.ts b/src/types.ts index 693133c..b4f47d0 100644 --- a/src/types.ts +++ b/src/types.ts @@ -27,6 +27,7 @@ export type ExportCollectionConfig = Record< export type CollectionExporterOptions = { excludeFields?: string[]; + linkedFields?: string[]; getKey?: (o: Item) => PrimaryKey; query?: Pick; prefix?: string; diff --git a/src/updateManager.ts b/src/updateManager.ts index dd3ed80..7b9b019 100644 --- a/src/updateManager.ts +++ b/src/updateManager.ts @@ -36,7 +36,7 @@ export class UpdateManager { // Only need to migrate if hash is different .andWhereNot('mv_hash', newHash) // And only if the previous hash is older than the current one - .andWhere('mv_ts', '<', isoTS) + .andWhere('mv_ts', '<', isoTS).orWhereNull('mv_ts') .forUpdate(); // This locks the row // If row is found, lock it