From d67f76580d25148615362dc8f6da3444ec17289e Mon Sep 17 00:00:00 2001 From: ezelsu Date: Mon, 24 Aug 2020 17:56:29 +0300 Subject: [PATCH 1/2] featt: algo recommendations for primitive types and specific words --- src/common/recommendation.ts | 2 +- .../services/deidentification.service.ts | 26 +++++++++---------- src/common/services/evaluation.service.ts | 4 +-- src/common/utils/fhir-util.ts | 20 ++++++++++++++ src/components/tables/FhirAttributeTable.vue | 18 +++++++------ src/store/fhirStore/index.ts | 23 ++++++++++------ 6 files changed, 61 insertions(+), 32 deletions(-) diff --git a/src/common/recommendation.ts b/src/common/recommendation.ts index 601d9e9..fc9c1fb 100644 --- a/src/common/recommendation.ts +++ b/src/common/recommendation.ts @@ -24,7 +24,7 @@ export const recommendation = { } }, parameterMappings: { - quasiRules: { + quasiRules: { // Redaction is always recommended with another algorithm, in case the attribute is required specificWords: { display: ['Redaction', 'Recoverable Substitution'], text: ['Redaction', 'Recoverable Substitution'], diff --git a/src/common/services/deidentification.service.ts b/src/common/services/deidentification.service.ts index badfc32..707bc66 100644 --- a/src/common/services/deidentification.service.ts +++ b/src/common/services/deidentification.service.ts @@ -164,8 +164,8 @@ export class DeidentificationService { while (i < paths.length) { key += '.' + paths[i++]; } - if (this.parameterMappings[key].name === 'Pass Through' || this.parameterMappings[key].name === 'Generalization' || - (this.parameterMappings[key].name === 'Substitution' && !environment.primitiveTypes[this.typeMappings[key]].regex + if (this.parameterMappings[key].name === environment.algorithms.PASS_THROUGH.name || this.parameterMappings[key].name === environment.algorithms.GENERALIZATION.name || + (this.parameterMappings[key].name === environment.algorithms.SUBSTITUTION.name && !environment.primitiveTypes[this.typeMappings[key]].regex && this.parameterMappings[key].lengthPreserved)) { keys.push(key); } @@ -192,13 +192,13 @@ export class DeidentificationService { const algorithm = this.parameterMappings[key]; [this.quasis, this.sensitives, this.identifiers] = [[], [], []]; switch (algorithm.name) { - case 'Pass Through': + case environment.algorithms.PASS_THROUGH.name: if (primitiveType === 'boolean') { if (!required) { this.identifiers.push(key.split('.').slice(2)); } this.canBeAnonymizedMore = false; - } else if (environment.primitiveTypes[primitiveType].supports.includes('Generalization')) { + } else if (environment.primitiveTypes[primitiveType].supports.includes(environment.algorithms.GENERALIZATION.name)) { this.parameterMappings[key] = environment.algorithms.GENERALIZATION; this.quasis.push(key.split('.').slice(2)); } else { @@ -206,7 +206,7 @@ export class DeidentificationService { this.quasis.push(key.split('.').slice(2)); } break; - case 'Generalization': + case environment.algorithms.GENERALIZATION.name: if (primitiveType === 'decimal') { // Decimal places of the floating number will be rounded if (this.parameterMappings[key].roundDigits) { this.parameterMappings[key].roundDigits--; @@ -263,7 +263,7 @@ export class DeidentificationService { } } break; - case 'Substitution': // data with regex is already filtered + case environment.algorithms.SUBSTITUTION.name: // data with regex is already filtered if (algorithm.lengthPreserved) { // anonymize again with fixed length this.parameterMappings[key] = environment.algorithms.SUBSTITUTION; this.parameterMappings[key].lengthPreserved = false; @@ -437,12 +437,12 @@ export class DeidentificationService { executeAlgorithm (key, parameters, data, primitiveType) { const regex = environment.primitiveTypes[primitiveType].regex; switch (parameters.name) { - case 'Pass Through': + case environment.algorithms.PASS_THROUGH.name: break; - case 'Redaction': + case environment.algorithms.REDACTION.name: this.identifiers.push(key.split('.').slice(2)); break; - case 'Substitution': + case environment.algorithms.SUBSTITUTION.name: if (regex) { data = new RandExp(regex).gen(); } else { @@ -450,10 +450,10 @@ export class DeidentificationService { .join( parameters.substitutionChar ); } break; - case 'Recoverable Substitution': + case environment.algorithms.RECOVERABLE_SUBSTITUTION.name: data = btoa(data); // recover function is atob(data) break; - case 'Fuzzing': + case environment.algorithms.FUZZING.name: data += this.getRandomFloat(-parameters.percentage, parameters.percentage); if (primitiveType === 'integer') { // A signed integer in the range −2,147,483,648..2,147,483,647 data = Math.round(data); @@ -463,7 +463,7 @@ export class DeidentificationService { data = Math.round(Math.abs(data)) ? Math.round(Math.abs(data)) : 1; } break; - case 'Generalization': + case environment.algorithms.GENERALIZATION.name: if (primitiveType === 'decimal') { // Decimal places of the floating number will be rounded const denary = Math.pow(10, parameters.roundDigits); data = parameters.roundedToFloor ? Math.floor(data * denary) / denary : Math.ceil(data * denary) / denary; @@ -496,7 +496,7 @@ export class DeidentificationService { } } break; - case 'Date Shifting': // Date will be shifted randomly within a range that you provide + case environment.algorithms.DATE_SHIFTING.name: // Date will be shifted randomly within a range that you provide if (primitiveType === 'time') { // HH:mm:ss ['Hours', 'Minutes', 'Seconds'] let tempDate = moment(data, 'HH:mm:ss').toDate(); tempDate = this.getRandomDate(tempDate, parameters.dateUnit, parameters.range); diff --git a/src/common/services/evaluation.service.ts b/src/common/services/evaluation.service.ts index 0b46fbe..18db82a 100644 --- a/src/common/services/evaluation.service.ts +++ b/src/common/services/evaluation.service.ts @@ -32,8 +32,8 @@ export class EvaluationService { while (i < paths.length) { key += '.' + paths[i++]; } - if (parameterMappings[key].name === 'Pass Through' || parameterMappings[key].name === 'Generalization' || - (parameterMappings[key].name === 'Substitution' && !environment.primitiveTypes[typeMappings[key]].regex + if (parameterMappings[key].name === environment.algorithms.PASS_THROUGH.name || parameterMappings[key].name === environment.algorithms.GENERALIZATION.name || + (parameterMappings[key].name === environment.algorithms.SUBSTITUTION.name && !environment.primitiveTypes[typeMappings[key]].regex && parameterMappings[key].lengthPreserved)) { this.riskyQuasis.push(key); } diff --git a/src/common/utils/fhir-util.ts b/src/common/utils/fhir-util.ts index 96db7e9..f654f41 100644 --- a/src/common/utils/fhir-util.ts +++ b/src/common/utils/fhir-util.ts @@ -1,4 +1,5 @@ import {environment} from '@/common/environment'; +import { recommendation } from '@/common/recommendation' import {FhirService} from '@/common/services/fhir.service'; import {Utils} from '@/common/utils/util'; import fhirStore from '@/store/fhirStore'; @@ -119,4 +120,23 @@ export class FHIRUtils { return tree; } + static recommendedAlgorithm (word: string, type: string, required: boolean, isQuasi: boolean) { + if (!isQuasi) { + // todo handle sensitive attribute algo recommendations + return environment.algorithms.SENSITIVE; + } else { + const algoOptions = recommendation.parameterMappings.quasiRules.specificWords[word] ? + recommendation.parameterMappings.quasiRules.specificWords[word] : recommendation.parameterMappings.quasiRules.primitiveTypes[type]; + if (!algoOptions) { + return environment.algorithms.PASS_THROUGH; + } else if (required && algoOptions.length > 1) { + const algoKey = Object.keys(environment.algorithms).find(key => environment.algorithms[key].name === algoOptions[1]); + return environment.algorithms[algoKey ? algoKey : 'PASS_THROUGH']; + } else { + const algoKey = Object.keys(environment.algorithms).find(key => environment.algorithms[key].name === algoOptions[0]); + return environment.algorithms[algoKey ? algoKey : 'PASS_THROUGH']; + } + } + } + } diff --git a/src/components/tables/FhirAttributeTable.vue b/src/components/tables/FhirAttributeTable.vue index 87bfe31..7ae4710 100644 --- a/src/components/tables/FhirAttributeTable.vue +++ b/src/components/tables/FhirAttributeTable.vue @@ -124,19 +124,19 @@
+ @input="onAttributeTypeSelected(prop, attributeTypes.ID)" :disable="prop.node.required" />
+ @input="onAttributeTypeSelected(prop, attributeTypes.QUASI)" />
+ @input="onAttributeTypeSelected(prop, attributeTypes.SENSITIVE)" />
+ @input="onAttributeTypeSelected(prop, attributeTypes.INSENSITIVE)" />
@@ -314,12 +314,14 @@ export default class FhirAttributeTable extends Vue { update(_ => this.fhirResourceOptions = this.fhirResourceList.filter(v => v.toLowerCase().includes(val.toLowerCase()))) } - onAttributeTypeSelected (prop: string, val: string) { - this.attributeMappings[prop] = val; + onAttributeTypeSelected (prop, val: string) { + const splitted = prop.key.split('.'); + const word = splitted[splitted.length - 1]; + this.attributeMappings[prop.key] = val; if (val === this.attributeTypes.SENSITIVE) { - this.parameterMappings[prop] = JSON.parse(JSON.stringify(environment.algorithms.SENSITIVE)); + this.parameterMappings[prop.key] = FHIRUtils.recommendedAlgorithm(word, prop.node.type, prop.node.required, false) } else if (val === this.attributeTypes.QUASI) { - this.parameterMappings[prop] = JSON.parse(JSON.stringify(environment.algorithms.PASS_THROUGH)); + this.parameterMappings[prop.key] = FHIRUtils.recommendedAlgorithm(word, prop.node.type, prop.node.required, true) } } diff --git a/src/store/fhirStore/index.ts b/src/store/fhirStore/index.ts index d4ce999..6f9cba1 100644 --- a/src/store/fhirStore/index.ts +++ b/src/store/fhirStore/index.ts @@ -1,10 +1,10 @@ -import { FhirService } from '@/common/services/fhir.service' -import { environment } from '@/common/environment' -import { recommendation } from '@/common/recommendation' -import { FHIRUtils } from '@/common/utils/fhir-util' +import {FhirService} from '@/common/services/fhir.service' +import {environment} from '@/common/environment' +import {recommendation} from '@/common/recommendation' +import {FHIRUtils} from '@/common/utils/fhir-util' import {EvaluationService} from '@/common/services/evaluation.service'; -import { VuexStoreUtil as types } from '@/common/utils/vuex-store-util' -import { LocalStorageUtil as localStorageKey } from '@/common/utils/local-storage-util' +import {VuexStoreUtil as types} from '@/common/utils/vuex-store-util' +import {LocalStorageUtil as localStorageKey} from '@/common/utils/local-storage-util' import i18n from '@/i18n'; const fhirStore = { @@ -33,8 +33,10 @@ const fhirStore = { state.typeMappings[tmpObj.value] = tmpObj.type; } if (tmpObj.value && FHIRUtils.isPrimitive(tmpObj, state.typeMappings) && !state.attributeMappings[tmpObj.value]) { - const resource = tmpObj.value.split('.')[0]; - const key = tmpObj.value.split('.').slice(2).join('.'); + const splitted = tmpObj.value.split('.'); + const resource = splitted[0]; + const key = splitted.slice(2).join('.'); + const word = splitted[splitted.length - 1]; if (recommendation.attributeMappings[resource][key]) { if (!tmpObj.required || (tmpObj.required && recommendation.attributeMappings[resource][key] !== environment.attributeTypes.ID)) { // if no conflict with required value, assign recommendation @@ -48,6 +50,11 @@ const fhirStore = { // if no recommendation exists for the current attribute, assign INSENSITIVE state.attributeMappings[tmpObj.value] = environment.attributeTypes.INSENSITIVE; } + if (state.attributeMappings[tmpObj.value] === environment.attributeTypes.QUASI) { + state.parameterMappings[tmpObj.value] = FHIRUtils.recommendedAlgorithm(word, tmpObj.type, tmpObj.required, true); + } else if (state.attributeMappings[tmpObj.value] === environment.attributeTypes.SENSITIVE) { + state.parameterMappings[tmpObj.value] = FHIRUtils.recommendedAlgorithm(word, tmpObj.type, tmpObj.required, false); + } } if (tmpObj.value && tmpObj.required && !state.requiredElements.includes(tmpObj.value)) { state.requiredElements.push(tmpObj.value); From 9f85ca7bb0d70ce5fba44b314daa956d6f318181 Mon Sep 17 00:00:00 2001 From: ezelsu Date: Tue, 15 Sep 2020 16:46:55 +0300 Subject: [PATCH 2/2] :bug: fix: eq classes size check --- src/common/services/deidentification.service.ts | 8 +++++++- src/components/Deidentifier.vue | 4 ++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/common/services/deidentification.service.ts b/src/common/services/deidentification.service.ts index 707bc66..bd246f2 100644 --- a/src/common/services/deidentification.service.ts +++ b/src/common/services/deidentification.service.ts @@ -105,6 +105,7 @@ export class DeidentificationService { while (this.canBeAnonymizedMore) { let parametersChanged = false; let eqClassesSmall = false; + let eqClassesEnough = false; equivalenceClasses.forEach(eqClass => { if (eqClass.length < kValue) { // k-anonymity if (!parametersChanged) { // change de-identification parameters to anonymize records more @@ -128,13 +129,18 @@ export class DeidentificationService { // anonymize records more which are not l-diverse eqClass = eqClass.map(entry => this.changeAttributes(resource + '.' + profile, entry.resource)); eqClassesSmall = true; + } else { + eqClassesEnough = true; } }); + } else { + eqClassesEnough = true; } }); equivalenceClasses = this.generateEquivalenceClasses(resource, quasiKey, anonymizedData); anonymizedData = [].concat(...equivalenceClasses); - if (eqClassesSmall) { + if (eqClassesEnough && !eqClassesSmall) { + // if all eq classes are large enough, exit loop break; } } diff --git a/src/components/Deidentifier.vue b/src/components/Deidentifier.vue index c00d469..5aa1fd4 100644 --- a/src/components/Deidentifier.vue +++ b/src/components/Deidentifier.vue @@ -800,11 +800,11 @@ export default class Deidentifier extends Mixins(StatusMixin) { } .target-repo-card { width: 700px - max-width: 80vw + max-width: 80vw !important } .json-resources-card { width: 900px - max-width: 100vw + max-width: 100vw !important } .json-resources-card-section { max-height: 70vh