Fix importing ordered items

CHANGELOG.md

@@ -1,4 +1,24 @@
-# Version 1.6.2
+# Version 1.6.3
+
+  - Add `linkedFields` to fix inserting ordered items that might depend on each other.
+    - Specifically fixes importing of flow operations, update the directus_config and add `linkedFields`.
+    ```javascript
+    {
+      directus_operations: {
+        watch: ['operations'],
+        excludeFields: ['user_created'],
+        linkedFields: ['resolve', 'reject'],
+        query: {
+          filter: {
+            flow: { trigger: { _neq: 'webhook' } },
+          },
+        },
+      },
+    }
+    ```
+  - Also fix auto IMPORT issue when mv_ts is null.
+
+## Version 1.6.2
 
   - (postgres only) Fixed import issue due to default_value containing functions for auto increment
 

install/schema-sync/config.js

@@ -5,6 +5,7 @@
  *  collectionName:
  * 		watch: array of events to watch for changes, eg. 'posts.items',
  * 		excludeFields: (optional) array of fields to exclude from the export,
+ * 		m2oFields: (optional) array of fields to treat as many-to-one relationships for hierarchy, eg. ['parent'],
  * 		getKey: (optional) function to get the key for the item, defaults to primary key found on schema,
  * 		query: (optional) query to use when exporting the collection, valid options are: (limit=-1 | filter | sort)
  * 		prefix: (optional) prefix the exported json file with this string (useful for separating test data from production data)
@@ -16,6 +17,7 @@ export const syncCustomCollections = {
 	posts: {
 		watch: ['posts.items'],
 		excludeFields: ['user_created', 'user_updated'],
+		m2oFields: ['parent'],
 		query: {
 			filter: {
 				shared: { _eq: true }

install/schema-sync/directus_config.js

@@ -66,6 +66,7 @@ export const syncDirectusCollections = {
 	directus_operations: {
 		watch: ['operations'],
 		excludeFields: ['user_created'],
+		linkedFields: ['resolve', 'reject'],
 		query: {
 			filter: {
 				flow: { trigger: { _neq: 'webhook' } },

src/collectionExporter.ts

@@ -64,7 +64,18 @@ class CollectionExporter implements IExporter {
 		await writeFile(this.filePath, json);
 	}
 
+	protected _settings: {
+		inclFields: Array<string>;
+		exclFields: Array<string>;
+		linkedFields: NonNullable<CollectionExporterOptions['linkedFields']>
+		getKey: (o: Item) => PrimaryKey;
+		getPrimary: (o: Item) => PrimaryKey;
+		query: Query;
+		queryWithPrimary: Query;
+	} | null = null;
 	protected async settings() {
+		if (this._settings) return this._settings;
+
 		const itemsSvc = await this._getService()
 		const schema = itemsSvc.schema.collections[this.collection]
 
@@ -95,9 +106,10 @@ class CollectionExporter implements IExporter {
 
 		const queryWithPrimary: Query = exclFields.includes(schema.primary) ? { ...query, fields: [...inclFields, schema.primary] } : query;
 
-		return {
+		return this._settings = {
 			inclFields,
 			exclFields,
+			linkedFields: this.options.linkedFields || [],
 			getKey,
 			getPrimary,
 			query,
@@ -125,6 +137,46 @@ class CollectionExporter implements IExporter {
 		return JSON.stringify(sortObject(items), null, 2);
 	}
 
+
+	/**
+	 * Orders items so that items that are linked are inserted after the items they reference
+	 * Only works with items that have a primary key
+	 * Assumes items not in given items list are already in the database
+	 * @param items 
+	 * @returns 
+	 */
+	protected async sortbyIfLinked(items: Array<Item>) {
+		const { getPrimary, linkedFields } = await this.settings();
+		if (!linkedFields.length) return false;
+
+		const itemsMap = items.reduce((map, o) => {
+			o.__dependents = [];
+			map[getPrimary(o)] = o;
+			return map;
+		}, {} as Record<PrimaryKey, Item>);
+
+		let hasDependents = false;
+		items.forEach(o => {
+			for (const fieldName of linkedFields) {
+				const value = o[fieldName];
+				if (value && itemsMap[value]) {
+					hasDependents = true;
+					itemsMap[value].__dependents.push(o);
+				}
+			}
+		});
+
+		items.sort((a, b) => this.countDependents(b) - this.countDependents(a));
+		items.forEach(o => delete o.__dependents);
+
+		return true;
+	}
+	// Recursively count dependents
+	private countDependents(o: any): number {
+		if (!o.__dependents.length) return 0;
+		return o.__dependents.reduce((acc, o) => acc + this.countDependents(o), o.__dependents.length);
+	}
+
 	public async loadJSON(json: JSONString | null, merge = false) {
 		if (!json) return null;
 		const loadedItems = JSON.parse(json) as Array<Item>;
@@ -182,7 +234,13 @@ class CollectionExporter implements IExporter {
 		// Insert
 		if (toInsert.length > 0) {
 			this.logger.debug(`Inserting ${toInsert.length} x ${this.collection} items`);
-			await itemsSvc.createMany(toInsert);
+			if (await this.sortbyIfLinked(toInsert)) {
+				for (const item of toInsert) {
+					await itemsSvc.createOne(item);
+				}
+			} else {
+				await itemsSvc.createMany(toInsert);
+			}
 		}
 
 		// Update

src/default_config.ts

@@ -70,6 +70,7 @@ export const syncDirectusCollections: ExportCollectionConfig = {
 	directus_operations: {
 		watch: ['operations'],
 		excludeFields: ['user_created'],
+		linkedFields: ['resolve', 'reject'],
 	},
 	directus_translations: {
 		watch: ['translations'],

src/types.ts

@@ -27,6 +27,7 @@ export type ExportCollectionConfig = Record<
 
 export type CollectionExporterOptions = {
 	excludeFields?: string[];
+	linkedFields?: string[];
 	getKey?: (o: Item) => PrimaryKey;
 	query?: Pick<Query, 'filter'|'sort'|'limit'>;
 	prefix?: string;

src/updateManager.ts

@@ -36,7 +36,7 @@ export class UpdateManager {
         // Only need to migrate if hash is different
         .andWhereNot('mv_hash', newHash)
         // And only if the previous hash is older than the current one
-        .andWhere('mv_ts', '<', isoTS)
+        .andWhere('mv_ts', '<', isoTS).orWhereNull('mv_ts')
         .forUpdate(); // This locks the row
 
       // If row is found, lock it