avniproject/avni-client#1309 - migrate all transaction entities toget…

…her in single pass. cache concepts. log time taken. reduced schema version as migration will take place in single pass.
avniproject · Feb 26, 2024 · 180550a · 180550a
1 parent 9b3e627
commit 180550a
Show file tree

Hide file tree

Showing 3 changed files with 59 additions and 37 deletions.
diff --git a/src/Schema.js b/src/Schema.js
@@ -90,6 +90,7 @@ import CustomDashboardCache from './CustomDashboardCache';
 import DefinedObjectSchema from "./framework/DefinedObjectSchema";
 import MigrationsHelper from "./MigrationsHelper";
 import MetaDataService from "./service/MetaDataService";
+import moment from "moment";
 
 const entities = [
     DashboardFilter,
@@ -258,58 +259,76 @@ function shouldFlush(numberOfRecords) {
     return (numberOfRecords % batchSize) === (batchSize - 1);
 }
 
-function migrateGeoLocation(oldDB, newDB) {
-    flush(newDB);
-    let recordCounter = 0;
-
-    MetaDataService.forEachPointField((field, schemaName) => {
-        console.log(`schema: ${schemaName}, field: ${field}`);
-        newDB.objects(schemaName).forEach((newDbParentEntity) => {
-            if (shouldFlush(recordCounter)) {
-                flush(newDB);
-            }
-
-            const oldFieldValue = oldDB.objects(schemaName).filtered(`uuid = "${newDbParentEntity.uuid}"`)[0][field];
-            if (!_.isNil(oldFieldValue)) {
-                newDbParentEntity[field] = {x: oldFieldValue.x, y: oldFieldValue.y};
-            }
-            recordCounter++;
-        });
+export function createTransactionDataMapForEmbeddedFields() {
+    const map = new Map();
+    MetaDataService.forEachPointField((fieldName, schemaName) => {
+        if (map.has(schemaName)) {
+            map.get(schemaName).push({fieldName, fieldType: "Point"});
+        } else {
+            map.set(schemaName, [{fieldName, fieldType: "Point"}]);
+        }
     });
-    flush(newDB);
-    newDB.deleteModel("Point");
+    MetaDataService.forEachObservationField((fieldName, schemaName) => {
+        if (map.has(schemaName)) {
+            map.get(schemaName).push({fieldName, fieldType: "Obs"});
+        } else {
+            map.set(schemaName, [{fieldName, fieldType: "Obs"}]);
+        }
+    });
+    return map;
 }
 
-function migrateObservationsToEmbedded(oldDB, newDB) {
+function migrateAllEmbeddedForTxnData(oldDB, newDB) {
+    const startTime = new Date();
     flush(newDB);
-    let recordCounter = 0;
-    MetaDataService.forEachObservationField((observationField, schemaName) => {
-        console.log(`schema: ${schemaName}, field: ${observationField}`);
+    const map = createTransactionDataMapForEmbeddedFields();
 
+    let recordCounter = 0;
+    const conceptMap = new Map();
+    map.forEach((fields, schemaName) => {
+        console.log(`schema: ${schemaName}, fields: ${fields.length}`);
         newDB.objects(schemaName).forEach((newDbParentEntity) => {
             if (shouldFlush(recordCounter)) {
                 flush(newDB);
             }
-
-            const newList = [];
-            const oldFieldValues = oldDB.objects(schemaName).filtered(`uuid = "${newDbParentEntity.uuid}"`)[0][observationField];
-            oldFieldValues.forEach((oldFieldValue) => {
-                const newConcept = newDB.objects("Concept").filtered(`uuid = "${oldFieldValue.concept.uuid}"`)[0];
-                newList.push({concept: newConcept, valueJSON: oldFieldValue.valueJSON});
+            fields.forEach((field) => {
+                const oldEntity = oldDB.objects(schemaName).filtered(`uuid = "${newDbParentEntity.uuid}"`)[0];
+                const oldValue = oldEntity[field.fieldName];
+                if (!_.isNil(oldValue)) {
+                    if (field.fieldType === "Point")
+                        newDbParentEntity[field.fieldName] = {x: oldValue.x, y: oldValue.y};
+                    else {
+                        const newObsList = [];
+                        oldValue.forEach((oldItemValue) => {
+                            let newConcept = conceptMap.get(oldItemValue.concept.uuid);
+                            if (_.isNil(newConcept)) {
+                                newConcept = newDB.objects("Concept").filtered(`uuid = "${oldItemValue.concept.uuid}"`)[0];
+                                conceptMap.set(oldItemValue.concept.uuid, newConcept);
+                            }
+                            newObsList.push({
+                                concept: newConcept,
+                                valueJSON: oldItemValue.valueJSON
+                            });
+                        });
+                        newDbParentEntity[field.fieldName] = newObsList;
+                    }
+                }
             });
-            newDbParentEntity[observationField] = newList;
-
             recordCounter++;
         });
     });
     flush(newDB);
+    newDB.deleteModel("Point");
     newDB.deleteModel("Observation");
+    const endTime = new Date();
+    const diff = moment(endTime).diff(startTime, "seconds", true);
+    console.log("Total Time Taken", diff, "seconds");
 }
 
 function createRealmConfig() {
     return {
         //order is important, should be arranged according to the dependency
-        schemaVersion: 186,
+        schemaVersion: 185,
         onMigration: function (oldDB, newDB) {
             console.log("[AvniModels.Schema]", `Running migration with old schema version: ${oldDB.schemaVersion} and new schema version: ${newDB.schemaVersion}`);
             if (oldDB.schemaVersion < 10) {
@@ -949,10 +968,7 @@ function createRealmConfig() {
                 migrateEmbeddedObjects(oldDB, newDB);
             }
             if (oldDB.schemaVersion < 185) {
-                migrateGeoLocation(oldDB, newDB);
-            }
-            if (oldDB.schemaVersion < 186) {
-                migrateObservationsToEmbedded(oldDB, newDB);
+                migrateAllEmbeddedForTxnData(oldDB, newDB);
             }
         },
     };

diff --git a/test/SchemaTest.js b/test/SchemaTest.js
@@ -0,0 +1,7 @@
+import {assert} from "chai";
+import {createTransactionDataMapForEmbeddedFields} from "../src/Schema";
+
+it('should createTransactionDataMapForEmbeddedFields', function () {
+    const map = createTransactionDataMapForEmbeddedFields();
+    assert.equal(map.get("Encounter").length, 4);
+});
diff --git a/test/ref/TestObservationFactory.js b/test/ref/TestObservationFactory.js
@@ -1,4 +1,3 @@
-import _ from "lodash";
 import {Observation} from "../../src";
 
 class TestObservationFactory {