From 617ade8b9004b785c34e9dd0ee421e8c48f5b89c Mon Sep 17 00:00:00 2001 From: Joy A Date: Thu, 28 Sep 2023 15:15:08 +0530 Subject: [PATCH 1/4] #1107 | Suhas/Joy | Remove structured-clone serialization of CustomDashboardCache fields Removed @ungap/structured-clone as a dependency. Added General.deepOmit function to remove specific key(s) in a nested object. Helps prevent cyclic reference errors during JSON.stringify. --- packages/openchs-android/package-lock.json | 31 +++++-------------- packages/openchs-android/package.json | 3 +- .../customDashboard/CustomDashboardActions.js | 3 +- .../action/mydashboard/FiltersActionsV2.js | 26 +++++++++------- .../openchs-android/src/utility/General.js | 17 ++++++++++ 5 files changed, 41 insertions(+), 39 deletions(-) diff --git a/packages/openchs-android/package-lock.json b/packages/openchs-android/package-lock.json index 879408101..4ce54f1f3 100644 --- a/packages/openchs-android/package-lock.json +++ b/packages/openchs-android/package-lock.json @@ -18,7 +18,6 @@ "@react-native-cookies/cookies": "^6.2.1", "@react-native-firebase/analytics": "15.2.0", "@react-native-firebase/app": "15.2.0", - "@ungap/structured-clone": "1.2.0", "amazon-cognito-identity-js": "6.3.1", "avni-health-modules": "^0.0.19", "base-64": "^1.0.0", @@ -34,7 +33,7 @@ "lodash": "4.17.21", "moment": "2.29.4", "native-base": "3.4.9", - "openchs-models": "1.30.69", + "openchs-models": "1.30.71", "prop-types": "15.8.1", "react": "18.2.0", "react-native": "0.72.3", @@ -5092,11 +5091,6 @@ "resolved": "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-21.0.0.tgz", "integrity": "sha512-iO9ZQHkZxHn4mSakYV0vFHAVDyEOIJQrV2uZ06HxEPcx+mt8swXoZHIbaaJ2crJYFfErySgktuTZ3BeLz+XmFA==" }, - "node_modules/@ungap/structured-clone": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz", - "integrity": "sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==" - }, "node_modules/@webassemblyjs/ast": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.9.0.tgz", @@ -16437,12 +16431,9 @@ } }, "node_modules/openchs-models": { - "version": "1.30.69", - "resolved": "https://registry.npmjs.org/openchs-models/-/openchs-models-1.30.69.tgz", - "integrity": "sha512-zMrdlpd4AyoYPeRv5sklp+TRcoY9QQszeXKqUQXaukMPqZvXuVP7rUAvqeHpSYmzVwsKCzGlnT9GqqouKKSaTg==", - "dependencies": { - "@ungap/structured-clone": "^1.2.0" - }, + "version": "1.30.71", + "resolved": "https://registry.npmjs.org/openchs-models/-/openchs-models-1.30.71.tgz", + "integrity": "sha512-ggakdscwcuB6RBGuJUTBqOxcRlAq6F3VbXFFCUqTU/JAmsCi5Usv65qlVfZPjUiaDvHox8BUsoBB2VYJGkiUPQ==", "peerDependencies": { "lodash": "*", "moment": "*" @@ -26319,11 +26310,6 @@ "resolved": "https://registry.npmjs.org/@types/yargs-parser/-/yargs-parser-21.0.0.tgz", "integrity": "sha512-iO9ZQHkZxHn4mSakYV0vFHAVDyEOIJQrV2uZ06HxEPcx+mt8swXoZHIbaaJ2crJYFfErySgktuTZ3BeLz+XmFA==" }, - "@ungap/structured-clone": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.2.0.tgz", - "integrity": "sha512-zuVdFrMJiuCDQUMCzQaD6KL28MjnqqN8XnAqiEq9PNm/hCPTSGfrXCOfwj1ow4LFb/tNymJPwsNbVePc1xFqrQ==" - }, "@webassemblyjs/ast": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.9.0.tgz", @@ -35145,12 +35131,9 @@ } }, "openchs-models": { - "version": "1.30.69", - "resolved": "https://registry.npmjs.org/openchs-models/-/openchs-models-1.30.69.tgz", - "integrity": "sha512-zMrdlpd4AyoYPeRv5sklp+TRcoY9QQszeXKqUQXaukMPqZvXuVP7rUAvqeHpSYmzVwsKCzGlnT9GqqouKKSaTg==", - "requires": { - "@ungap/structured-clone": "^1.2.0" - } + "version": "1.30.71", + "resolved": "https://registry.npmjs.org/openchs-models/-/openchs-models-1.30.71.tgz", + "integrity": "sha512-ggakdscwcuB6RBGuJUTBqOxcRlAq6F3VbXFFCUqTU/JAmsCi5Usv65qlVfZPjUiaDvHox8BUsoBB2VYJGkiUPQ==" }, "opencollective-postinstall": { "version": "2.0.3", diff --git a/packages/openchs-android/package.json b/packages/openchs-android/package.json index 5d81284ea..3574de3e3 100644 --- a/packages/openchs-android/package.json +++ b/packages/openchs-android/package.json @@ -42,7 +42,6 @@ "@react-native-cookies/cookies": "^6.2.1", "@react-native-firebase/analytics": "15.2.0", "@react-native-firebase/app": "15.2.0", - "@ungap/structured-clone": "1.2.0", "amazon-cognito-identity-js": "6.3.1", "avni-health-modules": "^0.0.19", "base-64": "^1.0.0", @@ -58,7 +57,7 @@ "lodash": "4.17.21", "moment": "2.29.4", "native-base": "3.4.9", - "openchs-models": "1.30.69", + "openchs-models": "1.30.71", "prop-types": "15.8.1", "react": "18.2.0", "react-native": "0.72.3", diff --git a/packages/openchs-android/src/action/customDashboard/CustomDashboardActions.js b/packages/openchs-android/src/action/customDashboard/CustomDashboardActions.js index fcdb582cf..0e206c581 100644 --- a/packages/openchs-android/src/action/customDashboard/CustomDashboardActions.js +++ b/packages/openchs-android/src/action/customDashboard/CustomDashboardActions.js @@ -8,7 +8,6 @@ import General from "../../utility/General"; import DashboardFilterService from "../../service/reports/DashboardFilterService"; import CustomDashboardCacheService from '../../service/CustomDashboardCacheService'; import CryptoUtils from '../../utility/CryptoUtils'; -import {serialize} from '@ungap/structured-clone'; class CustomDashboardActions { @@ -58,7 +57,7 @@ class CustomDashboardActions { const dashboardFilterService = context.get(DashboardFilterService); const customDashboardCacheService = context.get(CustomDashboardCacheService); const filterConfigs = dashboardFilterService.getFilterConfigsForDashboard(newState.activeDashboardUUID); - let filterConfigsJSON = JSON.stringify(serialize(filterConfigs)); + let filterConfigsJSON = JSON.stringify(filterConfigs); let filterConfigsChecksum = CryptoUtils.computeHash(filterConfigsJSON); const cachedData = customDashboardCacheService.fetchCachedData(newState.activeDashboardUUID, filterConfigsChecksum); newState.filterConfigsChecksum = cachedData.getChecksum(); diff --git a/packages/openchs-android/src/action/mydashboard/FiltersActionsV2.js b/packages/openchs-android/src/action/mydashboard/FiltersActionsV2.js index 08add08ee..e4c74fe5d 100644 --- a/packages/openchs-android/src/action/mydashboard/FiltersActionsV2.js +++ b/packages/openchs-android/src/action/mydashboard/FiltersActionsV2.js @@ -5,7 +5,7 @@ import {CustomDashboardActions} from '../customDashboard/CustomDashboardActions' import CustomDashboardCacheService from '../../service/CustomDashboardCacheService'; import CryptoUtils from '../../utility/CryptoUtils'; -import {serialize} from '@ungap/structured-clone'; +import General from "../../utility/General"; class FiltersActionsV2 { static getInitialState() { @@ -26,7 +26,7 @@ class FiltersActionsV2 { const filterConfigs = dashboardFilterService.getFilterConfigsForDashboard(action.dashboardUUID); const filters = dashboardFilterService.getFilters(action.dashboardUUID); let newState = {...state, filterConfigs: filterConfigs, filters: filters, loading: false}; - let filterConfigsJSON = JSON.stringify(serialize(newState.filterConfigs)); + let filterConfigsJSON = JSON.stringify(newState.filterConfigs); newState.filterConfigsChecksum = CryptoUtils.computeHash(filterConfigsJSON); const cachedData = context.get(CustomDashboardCacheService).fetchCachedData(action.dashboardUUID, newState.filterConfigsChecksum); if(state.dashboardUUID !== action.dashboardUUID) { @@ -57,6 +57,8 @@ class FiltersActionsV2 { break; case CustomFilter.type.Address: + updatedValue = General.deepOmit(value, 'locationMappings'); //including locationMappings causes cyclical reference errors during JSON.stringify + break; case Concept.dataType.Subject: case Concept.dataType.Text : case Concept.dataType.Notes : @@ -136,13 +138,15 @@ class FiltersActionsV2 { break; default: let customConceptValue = [{value: currentFilterValue}]; - if(filterConfig.widget === CustomFilter.widget.Range) { - customConceptValue = [{dateType: filterConfig.type, + if(_.get(filterConfig, 'widget') === CustomFilter.widget.Range) { + customConceptValue = [{dateType: _.get(filterConfig, 'type'), minValue: currentFilterValue.minValue, maxValue: currentFilterValue.maxValue}]; } - selectedFilters.selectedCustomFilters = {...selectedFilters.selectedCustomFilters, - [filterConfig.observationBasedFilter.concept.name] : customConceptValue}; + if (!_.isEmpty(_.get(filterConfig, 'observationBasedFilter.concept.name'))) { + selectedFilters.selectedCustomFilters = {...selectedFilters.selectedCustomFilters, + [_.get(filterConfig, 'observationBasedFilter.concept.name')] : customConceptValue}; + } break; } @@ -158,7 +162,7 @@ class FiltersActionsV2 { const filledFilterValues = _.filter(Object.entries(selectedValues), ([, filterValue]) => !ModelGeneral.isDeepEmpty(filterValue)); //Check if there are errors in filter values specified filledFilterValues.forEach(([filterUUID, filterValue]) => { - const [success, message] = filterConfigs[filterUUID].validate(filterValue); + const [success, message] = filterConfigs[filterUUID].validate && filterConfigs[filterUUID].validate(filterValue) || [false, `validate for filterConfig ${filterUUID} not found`]; if (!success) newState.filterErrors[filterUUID] = message; }); @@ -181,10 +185,10 @@ class FiltersActionsV2 { } static createCustomDashboardCache(newState, dashboardUUID, transformedFilters, ruleInputArray) { - let selectValueJSON = JSON.stringify(serialize(newState.selectedValues)); - let filteredErrorsJSON = JSON.stringify(serialize(newState.filterErrors)); - let transformedFiltersJSON = JSON.stringify(serialize(transformedFilters)); - let ruleInputJSON = JSON.stringify(serialize({ruleInputArray: ruleInputArray})); + let selectValueJSON = JSON.stringify(newState.selectedValues); + let filteredErrorsJSON = JSON.stringify(newState.filterErrors); + let transformedFiltersJSON = JSON.stringify(transformedFilters); + let ruleInputJSON = JSON.stringify({ruleInputArray: ruleInputArray}); const customDashboardCache = CustomDashboardCache.create(dashboardUUID, newState.filterConfigsChecksum, new Date(), selectValueJSON, newState.filterApplied, filteredErrorsJSON, ruleInputJSON, transformedFiltersJSON); return customDashboardCache; diff --git a/packages/openchs-android/src/utility/General.js b/packages/openchs-android/src/utility/General.js index b53dc55fc..5f567bce3 100644 --- a/packages/openchs-android/src/utility/General.js +++ b/packages/openchs-android/src/utility/General.js @@ -322,6 +322,23 @@ class General { static clearClipboard() { Clipboard.setString(''); } + + //from https://stackoverflow.com/questions/39085399/lodash-remove-items-recursively + static deepOmit(obj, keysToOmit) { + const keysToOmitIndex = _.keyBy(Array.isArray(keysToOmit) ? keysToOmit : [keysToOmit] ); // create an index object of the keys that should be omitted + + function omitFromObject(obj) { // the inner function which will be called recursivley + return _.transform(obj, function(result, value, key) { // transform to a new object + if (key in keysToOmitIndex) { // if the key is in the index skip it + return; + } + + result[key] = _.isObject(value) ? omitFromObject(value) : value; // if the key is an object run it through the inner function - omitFromObject + }) + } + + return omitFromObject(obj); // return the inner function result + } } export default General; From e9ee8cfcd639d2b4ab3ba5b43f60c10f4fba3d99 Mon Sep 17 00:00:00 2001 From: Vivek Singh Date: Fri, 6 Oct 2023 15:32:08 +0530 Subject: [PATCH 2/4] avniproject/avni-server#621 - model version change for onMigration instead of migration function. --- packages/openchs-android/package-lock.json | 14 +++++++------- packages/openchs-android/package.json | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/openchs-android/package-lock.json b/packages/openchs-android/package-lock.json index 4ce54f1f3..7a43ab641 100644 --- a/packages/openchs-android/package-lock.json +++ b/packages/openchs-android/package-lock.json @@ -33,7 +33,7 @@ "lodash": "4.17.21", "moment": "2.29.4", "native-base": "3.4.9", - "openchs-models": "1.30.71", + "openchs-models": "1.30.72", "prop-types": "15.8.1", "react": "18.2.0", "react-native": "0.72.3", @@ -16431,9 +16431,9 @@ } }, "node_modules/openchs-models": { - "version": "1.30.71", - "resolved": "https://registry.npmjs.org/openchs-models/-/openchs-models-1.30.71.tgz", - "integrity": "sha512-ggakdscwcuB6RBGuJUTBqOxcRlAq6F3VbXFFCUqTU/JAmsCi5Usv65qlVfZPjUiaDvHox8BUsoBB2VYJGkiUPQ==", + "version": "1.30.72", + "resolved": "https://registry.npmjs.org/openchs-models/-/openchs-models-1.30.72.tgz", + "integrity": "sha512-1j0HY+LOMJvTH+fLvpXJ9u4OrxKhTCxwzUM7yt/2gw9ZdPM52Ez9bYUYfol44F8PgiF2grnTkd7mfrvB0V/C6w==", "peerDependencies": { "lodash": "*", "moment": "*" @@ -35131,9 +35131,9 @@ } }, "openchs-models": { - "version": "1.30.71", - "resolved": "https://registry.npmjs.org/openchs-models/-/openchs-models-1.30.71.tgz", - "integrity": "sha512-ggakdscwcuB6RBGuJUTBqOxcRlAq6F3VbXFFCUqTU/JAmsCi5Usv65qlVfZPjUiaDvHox8BUsoBB2VYJGkiUPQ==" + "version": "1.30.72", + "resolved": "https://registry.npmjs.org/openchs-models/-/openchs-models-1.30.72.tgz", + "integrity": "sha512-1j0HY+LOMJvTH+fLvpXJ9u4OrxKhTCxwzUM7yt/2gw9ZdPM52Ez9bYUYfol44F8PgiF2grnTkd7mfrvB0V/C6w==" }, "opencollective-postinstall": { "version": "2.0.3", diff --git a/packages/openchs-android/package.json b/packages/openchs-android/package.json index 3574de3e3..dcd8d3f23 100644 --- a/packages/openchs-android/package.json +++ b/packages/openchs-android/package.json @@ -57,7 +57,7 @@ "lodash": "4.17.21", "moment": "2.29.4", "native-base": "3.4.9", - "openchs-models": "1.30.71", + "openchs-models": "1.30.72", "prop-types": "15.8.1", "react": "18.2.0", "react-native": "0.72.3", From 5d31c0356016e3c2cd72427cab9f3011f7619b36 Mon Sep 17 00:00:00 2001 From: himeshr Date: Fri, 6 Oct 2023 19:26:30 +0530 Subject: [PATCH 3/4] #1125 | restore userInfo after fast sync db restore --- .../openchs-android/src/service/BackupRestoreRealm.js | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/packages/openchs-android/src/service/BackupRestoreRealm.js b/packages/openchs-android/src/service/BackupRestoreRealm.js index 57fccec17..461f9ecd4 100644 --- a/packages/openchs-android/src/service/BackupRestoreRealm.js +++ b/packages/openchs-android/src/service/BackupRestoreRealm.js @@ -16,6 +16,7 @@ import SubjectTypeService from "./SubjectTypeService"; import IndividualService from "./IndividualService"; import SubjectMigrationService from "./SubjectMigrationService"; import FormMappingService from "./FormMappingService"; +import UserInfoService from './UserInfoService'; const REALM_FILE_NAME = "default.realm"; const REALM_FILE_FULL_PATH = `${fs.DocumentDirectoryPath}/${REALM_FILE_NAME}`; @@ -85,6 +86,7 @@ export default class BackupRestoreRealmService extends BaseService { let downloadedFile = `${fs.DocumentDirectoryPath}/${General.randomUUID()}.zip`; let downloadedUncompressedDir = `${fs.DocumentDirectoryPath}/${General.randomUUID()}`; const prevSettings = settingsService.getSettings().clone(); + const prevUserInfo = UserInfo.fromResource({username: prevSettings.userId, organisationName: 'dummy', name: prevSettings.userId}); General.logInfo("BackupRestoreRealm", `To be downloaded file: ${downloadedFile}, Unzipped directory: ${downloadedUncompressedDir}, Realm file: ${REALM_FILE_FULL_PATH}`); @@ -143,6 +145,11 @@ export default class BackupRestoreRealmService extends BaseService { this._deleteUserInfoAndIdAssignment(); General.logDebug("BackupRestoreRealmService", "Deleted user info and id assignment"); }) + .then(() => { + this._restoreUserInfo(prevUserInfo); + General.logDebug("BackupRestoreRealmService", "Restoring prev userInfo to ensure we have user details" + + " immediately after fast sync restore"); + }) .then(() => { this._deleteUserGroups(); General.logDebug("BackupRestoreRealmService", "Deleted user groups"); @@ -217,6 +224,10 @@ export default class BackupRestoreRealmService extends BaseService { this.getService(SettingsService).saveOrUpdate(prevSettings); } + _restoreUserInfo(prevUserInfo) { + this.getService(UserInfoService).saveOrUpdate(prevUserInfo); + } + resetSyncForSubjectType(subjectType, db) { const formMappingsForSubjectType = this.getService(FormMappingService).getFormMappingsForSubjectType(subjectType).map(_.identity); _.forEach(formMappingsForSubjectType, (formMapping) => { From a0c2423b6385ccb913933f53ee2dafb98429a631 Mon Sep 17 00:00:00 2001 From: himeshr Date: Mon, 9 Oct 2023 11:32:07 +0530 Subject: [PATCH 4/4] Revert "#1125 | restore userInfo after fast sync db restore" This reverts commit 5d31c0356016e3c2cd72427cab9f3011f7619b36. --- .../openchs-android/src/service/BackupRestoreRealm.js | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/packages/openchs-android/src/service/BackupRestoreRealm.js b/packages/openchs-android/src/service/BackupRestoreRealm.js index 461f9ecd4..57fccec17 100644 --- a/packages/openchs-android/src/service/BackupRestoreRealm.js +++ b/packages/openchs-android/src/service/BackupRestoreRealm.js @@ -16,7 +16,6 @@ import SubjectTypeService from "./SubjectTypeService"; import IndividualService from "./IndividualService"; import SubjectMigrationService from "./SubjectMigrationService"; import FormMappingService from "./FormMappingService"; -import UserInfoService from './UserInfoService'; const REALM_FILE_NAME = "default.realm"; const REALM_FILE_FULL_PATH = `${fs.DocumentDirectoryPath}/${REALM_FILE_NAME}`; @@ -86,7 +85,6 @@ export default class BackupRestoreRealmService extends BaseService { let downloadedFile = `${fs.DocumentDirectoryPath}/${General.randomUUID()}.zip`; let downloadedUncompressedDir = `${fs.DocumentDirectoryPath}/${General.randomUUID()}`; const prevSettings = settingsService.getSettings().clone(); - const prevUserInfo = UserInfo.fromResource({username: prevSettings.userId, organisationName: 'dummy', name: prevSettings.userId}); General.logInfo("BackupRestoreRealm", `To be downloaded file: ${downloadedFile}, Unzipped directory: ${downloadedUncompressedDir}, Realm file: ${REALM_FILE_FULL_PATH}`); @@ -145,11 +143,6 @@ export default class BackupRestoreRealmService extends BaseService { this._deleteUserInfoAndIdAssignment(); General.logDebug("BackupRestoreRealmService", "Deleted user info and id assignment"); }) - .then(() => { - this._restoreUserInfo(prevUserInfo); - General.logDebug("BackupRestoreRealmService", "Restoring prev userInfo to ensure we have user details" + - " immediately after fast sync restore"); - }) .then(() => { this._deleteUserGroups(); General.logDebug("BackupRestoreRealmService", "Deleted user groups"); @@ -224,10 +217,6 @@ export default class BackupRestoreRealmService extends BaseService { this.getService(SettingsService).saveOrUpdate(prevSettings); } - _restoreUserInfo(prevUserInfo) { - this.getService(UserInfoService).saveOrUpdate(prevUserInfo); - } - resetSyncForSubjectType(subjectType, db) { const formMappingsForSubjectType = this.getService(FormMappingService).getFormMappingsForSubjectType(subjectType).map(_.identity); _.forEach(formMappingsForSubjectType, (formMapping) => {