From 128e71acf845a5b207ee0fb13499bc63ac043faa Mon Sep 17 00:00:00 2001 From: Mile Divovic Date: Tue, 31 Jan 2023 16:06:12 -0800 Subject: [PATCH 1/3] #1269 allow importing of taxonomies with remote server option --- src/components/import/ImportServer.vue | 224 ++++++++++++++++++++++--- src/components/import/ImportTabs.vue | 4 +- src/lode/components/List.vue | 2 +- src/mixins/import.js | 16 +- src/views/framework/Import.vue | 51 +++--- 5 files changed, 243 insertions(+), 54 deletions(-) diff --git a/src/components/import/ImportServer.vue b/src/components/import/ImportServer.vue index 7813eb3ed..60adc3b5c 100644 --- a/src/components/import/ImportServer.vue +++ b/src/components/import/ImportServer.vue @@ -33,7 +33,8 @@ type="url"> -
+
+ v-if="cassDirectories.length || cassFrameworks.length || cassTaxonomies.length">

- Found Frameworks + Found Frameworks + Found Taxonomies

+ :searchType="conceptMode ? 'taxonomy' : 'framework'" />
@@ -204,7 +206,10 @@
-
+
+ + +
+

+ Select the taxonomies to import. +

+
@@ -277,7 +304,7 @@
+ @click="conceptMode ? importCassTaxonomies() : importCassFrameworks()"> Import
@@ -330,10 +357,12 @@ export default { serverType: '', cassDirectories: [], cassFrameworks: [], + cassTaxonomies: [], remoteRepo: null, directoryThatsOpen: null, selectDirectory: 'all', selectedFrameworks: [], + selectedTaxonomies: [], directoryTrail: [] }; }, @@ -354,6 +383,9 @@ export default { }, searchTerm: function() { return this.$store.getters['app/searchTerm']; + }, + conceptMode: function() { + return this.$store.getters['editor/conceptMode']; } }, mounted: function() { @@ -393,6 +425,7 @@ export default { this.caseDocs.splice(0, this.caseDocs.length); this.cassDirectories.splice(0, this.cassDirectories.length); this.cassFrameworks.splice(0, this.cassFrameworks.length); + this.cassTaxonomies.splice(0, this.cassTaxonomies.length); if (this.serverType === 'cass') { this.cassDetectEndpoint(); } else if (this.serverType === 'case') { @@ -419,8 +452,8 @@ export default { this.$store.commit('editor/setFirstSearchProcessing', true); let me = this; let paramObj = {}; - paramObj.size = 50; - paramObj.sort = '[ { "name.keyword": {"order" : "asc"}} ]'; + paramObj.size = 10000; + paramObj.sort = this.conceptMode ? '[]' : '[ { "name.keyword": {"order" : "asc"}} ]'; let search = "(*)"; if (this.searchTerm) { search = this.searchTerm; @@ -432,14 +465,25 @@ export default { appLog(error); me.cassDirectories.splice(0, me.cassDirectories.length); }, paramObj); - EcFramework.search(this.remoteRepo, search, function(success) { - me.cassFrameworks.splice(0, me.cassFrameworks.length); - me.cassSearchSuccess(success, "framework"); - }, function(error) { - me.cassFrameworks.splice(0, me.cassFrameworks.length); - appLog(error); - me.cassSearchError(); - }, paramObj); + if (!this.conceptMode) { + EcFramework.search(this.remoteRepo, search, function(success) { + me.cassFrameworks.splice(0, me.cassFrameworks.length); + me.cassSearchSuccess(success, "framework"); + }, function(error) { + me.cassFrameworks.splice(0, me.cassFrameworks.length); + appLog(error); + me.cassSearchError(); + }, paramObj); + } else { + EcConceptScheme.search(this.remoteRepo, search, function(success) { + me.cassTaxonomies.splice(0, me.cassTaxonomies.length); + me.cassSearchSuccess(success, "taxonomy"); + }, function(error) { + me.cassTaxonomies.splice(0, me.cassTaxonomies.length); + appLog(error); + me.cassSearchError(); + }, paramObj); + } }, 1000), cassSearchError: function() { this.$store.commit('editor/setFirstSearchProcessing', false); @@ -457,6 +501,10 @@ export default { let message = success.length + " frameworks detected."; this.$store.commit('app/importStatus', message); this.$store.commit('app/importTransition', 'serverFrameworksDetected'); + } else if (objectType === 'taxonomy') { + let message = success.length + " taxonomies detected."; + this.$store.commit('app/importStatus', message); + this.$store.commit('app/importTransition', 'serverFrameworksDetected'); } for (let each in success) { success[each].loading = false; @@ -467,9 +515,60 @@ export default { this.cassDirectories.push(success[each]); } else if (objectType === "framework") { this.cassFrameworks.push(success[each]); + } else if (objectType === 'taxonomy') { + this.cassTaxonomies.push(success[each]); } } }, + importCassTaxonomies: function(dataArray) { + this.$store.commit('app/importTransition', 'importingCassFrameworks'); + if (dataArray) { + // User has clicked cancel on this import item + var localFirstIndex = dataArray[1]; + this.cassTaxonomies[localFirstIndex].loading = false; + this.cassTaxonomies[localFirstIndex].error = true; + } + for (var i = this.cassTaxonomies.length - 1; i >= 0; i--) { + if (!this.cassTaxonomies[i].checked) { + this.cassTaxonomies.splice(i, 1); + } else if (this.cassTaxonomies[i].success === false && this.cassTaxonomies[i].error === false) { + this.cassTaxonomies[i].loading = true; + } + } + let lis = 0; + let firstIndex = null; + for (var i = 0; i < this.cassTaxonomies.length; i++) { + if (this.cassTaxonomies[i].loading === true) { + lis++; + if (firstIndex == null) { + firstIndex = i; + } + } + } + if (lis === 0) { + this.$store.commit('app/importFramework', this.$store.getters['editor/framework']); + if (this.cassTaxonomies.length === 1) { + this.importSuccess(); + } else { + this.$store.commit('app/sortResults', { + id: 'lastEdited', + label: 'last modified' + }); + this.$router.push({name: "concepts"}); + } + this.$store.commit('app/importStatus', "Import finished."); + } else { + var me = this; + EcRepository.cache = {}; + EcConceptScheme.get(this.cassTaxonomies[firstIndex].shortId(), function(found) { + me.$store.commit('app/importStatus', 'taxonomy found...'); + me.showModal('duplicateOverwriteOnly', [[me.cassTaxonomies[firstIndex], firstIndex], found]); + }, function(notFound) { + me.$store.commit('app/importStatus', 'no match, saving new taxonomy...'); + me.continueCassTaxonomyImport([me.cassTaxonomies[firstIndex], firstIndex]); + }); + } + }, importCassFrameworks: function(dataArray) { this.$store.commit('app/importTransition', 'importingCassFrameworks'); if (dataArray) { @@ -515,7 +614,7 @@ export default { me.showModal('duplicateOverwriteOnly', [[me.cassFrameworks[firstIndex], firstIndex], found]); }, function(notFound) { me.$store.commit('app/importStatus', 'no match, saving new framework...'); - me.continueCassImport([me.cassFrameworks[firstIndex], firstIndex]); + me.continueCassFrameworkImport([me.cassFrameworks[firstIndex], firstIndex]); }); } }, @@ -563,7 +662,69 @@ export default { } }, null); }, - continueCassImport(dataArray) { + async continueCassTaxonomyImport(dataArray) { + var data = dataArray[0]; + var firstIndex = dataArray[1]; + var me = this; + let taxonomy = new EcConceptScheme(); + taxonomy.copyFrom(data); + delete taxonomy.owner; + delete taxonomy.reader; + if (EcIdentityManager.default.ids.length > 0) { + taxonomy.addOwner(EcIdentityManager.default.ids[0].ppk.toPk()); + } + taxonomy.id = taxonomy.shortId(); + taxonomy["schema:dateModified"] = new Date().toISOString(); + delete taxonomy.loading; + delete taxonomy.success; + delete taxonomy.error; + delete taxonomy.checked; + delete taxonomy.directory; + // To do: address directory + let toSave = []; + toSave.push(taxonomy); + let subObjects = []; + if (taxonomy['skos:hasTopConcept'] && taxonomy['skos:hasTopConcept']) { + subObjects = (await EcConcept.search(this.remoteRepo, 'skos:inScheme\\:"' + taxonomy.shortId() + '"', undefined, undefined, {size: 10000})).map(x => x.shortId()); + } + EcRepository.alwaysTryUrl = true; + this.precacheRemoteServer(subObjects, function() { + new EcAsyncHelper().each(subObjects, function(subId, done) { + EcRepository.get(subId, function(result) { + let type = "Ec" + result.type; + if (type === "EcRelation") { + type = "EcAlignment"; + } + let newObj = new window[type](); + newObj.copyFrom(result); + delete newObj.owner; + delete newObj.reader; + if (EcIdentityManager.default.ids.length > 0) { + newObj.addOwner(EcIdentityManager.default.ids[0].ppk.toPk()); + } + newObj.id = newObj.shortId(); + toSave.push(newObj); + done(); + }, done); + }, function(allDone) { + EcRepository.alwaysTryUrl = false; + me.repo.multiput(toSave, function() { + me.cassTaxonomies[firstIndex].loading = false; + me.cassTaxonomies[firstIndex].success = true; + me.$store.commit('editor/framework', taxonomy); + me.spitEvent("importFinished", taxonomy.shortId(), "importPage"); + me.importCassTaxonomies(); + }, function() { + me.cassTaxonomies[firstIndex].loading = false; + me.cassTaxonomies[firstIndex].error = true; + me.importCassTaxonomies(); + }); + }); + }, function(error) { + appError(error); + }); + }, + continueCassFrameworkImport(dataArray) { var data = dataArray[0]; var firstIndex = dataArray[1]; var me = this; @@ -654,9 +815,16 @@ export default { } }, selectAllFrameworks: function() { - for (let each in this.cassFrameworks) { - this.cassFrameworks[each].checked = true; - EcArray.setAdd(this.selectedFrameworks, this.cassFrameworks[each].id); + if (!this.conceptMode) { + for (let each in this.cassFrameworks) { + this.cassFrameworks[each].checked = true; + EcArray.setAdd(this.selectedFrameworks, this.cassFrameworks[each].id); + } + } else { + for (let each in this.cassTaxonomies) { + this.cassTaxonomies[each].checked = true; + EcArray.setAdd(this.selectedTaxonomies, this.cassTaxonomies[each].id); + } } }, caseDetectEndpoint: function() { @@ -871,8 +1039,20 @@ export default { } } }, + selectedTaxonomies: function() { + for (let each in this.cassTaxonomies) { + if (EcArray.has(this.selectedTaxonomies, this.cassTaxonomies[each].id)) { + this.cassTaxonomies[each].checked = true; + } else { + this.cassTaxonomies[each].checked = false; + } + } + }, cassFrameworks: function() { this.selectedFrameworks.splice(0, this.selectedFrameworks.length); + }, + cassTaxonomies: function() { + this.selectedTaxonomies.splice(0, 1); } } }; diff --git a/src/components/import/ImportTabs.vue b/src/components/import/ImportTabs.vue index 489225534..11dcdb540 100644 --- a/src/components/import/ImportTabs.vue +++ b/src/components/import/ImportTabs.vue @@ -37,7 +37,7 @@
+ v-if="!progressionMode">
@@ -107,7 +107,7 @@ Upload documents to transform into CaSS {{ queryParams.ceasnDataFields === 'true' ? 'Concept Schemes' : 'Taxonomies' }}. Upload documents to transform into CaSS Progression Models. diff --git a/src/lode/components/List.vue b/src/lode/components/List.vue index 06342c7ce..3aafefdc6 100644 --- a/src/lode/components/List.vue +++ b/src/lode/components/List.vue @@ -657,7 +657,7 @@ export default { var subLocalParamObj = Object.assign({}, me.paramObj); if (me.searchTerm != null && me.searchTerm !== "") { delete subLocalParamObj.sort; } subLocalParamObj.start = me.subStart; - if (subLocalParamObj.sort.indexOf("dcterms:title") !== -1) { + if (subLocalParamObj.sort && subLocalParamObj.sort.indexOf("dcterms:title") !== -1) { subLocalParamObj.sort = subLocalParamObj.sort.replace('dcterms:title', 'skos:prefLabel'); } var type = me.type === "Framework" ? "Competency" : "Concept"; diff --git a/src/mixins/import.js b/src/mixins/import.js index 331fd0595..b5d0af899 100644 --- a/src/mixins/import.js +++ b/src/mixins/import.js @@ -63,7 +63,7 @@ export default { } else if (val === 'duplicateOverwriteOnly') { if (data[1] && (!EcIdentityManager.default.ids || EcIdentityManager.default.ids.length === 0)) { // An owner is attached from the server-side import so it can be overwritten, just not edited - let type = data[1].subType === "Collection" ? "collection" : "framework"; + let type = data[1].subType === "Collection" ? "collection" : data[1] && data[1].type === "ConceptScheme" ? "taxonomy" : "framework"; params = { type: val, title: "Duplicate " + type, @@ -73,7 +73,11 @@ export default { return this.importJsonLd(data[0]); } if (this.serverType === "cass") { - return this.continueCassImport(data[0]); + if (!this.conceptMode) { + return this.continueCassFrameworkImport(data[0]); + } else { + return this.continueCassTaxonomyImport(data[0]); + } } return this.continueCaseImport(data[0]); }, @@ -89,7 +93,7 @@ export default { } }; } else { - let type = data[1] && data[1].subType === "Collection" ? "collection" : "framework"; + let type = data[1].subType === "Collection" ? "collection" : data[1] && data[1].type === "ConceptScheme" ? "taxonomy" : "framework"; params = { type: val, title: "Duplicate " + type, @@ -98,7 +102,11 @@ export default { if (this.importType === "url") { return this.importJsonLd(data[0]); } else if (this.serverType === "cass") { - return this.continueCassImport(data[0]); + if (!this.conceptMode) { + return this.continueCassFrameworkImport(data[0]); + } else { + return this.continueCassTaxonomyImport(data[0]); + } } return this.continueCaseImport(data[0]); }, diff --git a/src/views/framework/Import.vue b/src/views/framework/Import.vue index afbb0d1b3..9b21eabcb 100644 --- a/src/views/framework/Import.vue +++ b/src/views/framework/Import.vue @@ -858,42 +858,43 @@

Remote Server Import

- -

- If you know the URL of a IMS CASE Repository, such as OpenSalt, you can import published frameworks from that repository. -

-
-
    -
  • - A CASE framework cannot be imported if it uses API Key authentication. -
    • -
  • - This import maintains the URLs of the CASE frameworks and changes both the format and schema used to store the CASE frameworks in CaSS, but does not change any of the data. -
    • -
  • - After entering the endpoint below, you can select which frameworks you would like to import. -
    • -
  • - If you wish to edit the frameworks after importing, please be sure you are signed in. -
    • -
-
+

- If you know the URL of another CaSS Repository, you can import frameworks from that repository. + If you know the URL of another CaSS Repository, you can import taxonomiesframeworks from that repository.


  • - This import maintains the URLs of the CaSS frameworks but does not change any of the data besides the owner. If you are logged in, you will have ownership of the newly imported framework and be able to edit it. + This import maintains the URLs of the CaSS taxonomiesframeworks but does not change any of the data besides the owner. If you are logged in, you will have ownership of the newly imported taxonomyframework and be able to edit it.
  • - After entering the endpoint below, you can select which frameworks you would like to import. + After entering the endpoint below, you can select which taxonomiesframeworks you would like to import.
  • - If the other repository has directories, these will appear at the top of the list. You can click on a directory to view and select frameworks within it. + If the other repository has directories, these will appear at the top of the list. You can click on a directory to view and select taxonomiesframeworks within it.
  • - Use the search bar to find particular frameworks or directories. + Use the search bar to find particular taxonomiesframeworks or directories.
From 2c63502f5484b4264b303d5143aff2f77f0b244f Mon Sep 17 00:00:00 2001 From: Mile Divovic Date: Tue, 31 Jan 2023 16:13:27 -0800 Subject: [PATCH 2/3] fix typo --- src/components/import/ImportServer.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/import/ImportServer.vue b/src/components/import/ImportServer.vue index 60adc3b5c..7684af20c 100644 --- a/src/components/import/ImportServer.vue +++ b/src/components/import/ImportServer.vue @@ -684,7 +684,7 @@ export default { let toSave = []; toSave.push(taxonomy); let subObjects = []; - if (taxonomy['skos:hasTopConcept'] && taxonomy['skos:hasTopConcept']) { + if (taxonomy['skos:hasTopConcept'] && taxonomy['skos:hasTopConcept'].length) { subObjects = (await EcConcept.search(this.remoteRepo, 'skos:inScheme\\:"' + taxonomy.shortId() + '"', undefined, undefined, {size: 10000})).map(x => x.shortId()); } EcRepository.alwaysTryUrl = true; From ea2f5b980c0f5f2da534a70a7c10b5d100714702 Mon Sep 17 00:00:00 2001 From: Mile Divovic Date: Tue, 31 Jan 2023 16:32:52 -0800 Subject: [PATCH 3/3] deduplicate code --- src/components/import/ImportServer.vue | 140 ++++++++++--------------- 1 file changed, 57 insertions(+), 83 deletions(-) diff --git a/src/components/import/ImportServer.vue b/src/components/import/ImportServer.vue index 7684af20c..55f8e9613 100644 --- a/src/components/import/ImportServer.vue +++ b/src/components/import/ImportServer.vue @@ -662,32 +662,21 @@ export default { } }, null); }, - async continueCassTaxonomyImport(dataArray) { - var data = dataArray[0]; - var firstIndex = dataArray[1]; - var me = this; - let taxonomy = new EcConceptScheme(); - taxonomy.copyFrom(data); - delete taxonomy.owner; - delete taxonomy.reader; + cleanData(thing) { + delete thing.owner; + delete thing.reader; if (EcIdentityManager.default.ids.length > 0) { - taxonomy.addOwner(EcIdentityManager.default.ids[0].ppk.toPk()); + thing.addOwner(EcIdentityManager.default.ids[0].ppk.toPk()); } - taxonomy.id = taxonomy.shortId(); - taxonomy["schema:dateModified"] = new Date().toISOString(); - delete taxonomy.loading; - delete taxonomy.success; - delete taxonomy.error; - delete taxonomy.checked; - delete taxonomy.directory; - // To do: address directory - let toSave = []; - toSave.push(taxonomy); - let subObjects = []; - if (taxonomy['skos:hasTopConcept'] && taxonomy['skos:hasTopConcept'].length) { - subObjects = (await EcConcept.search(this.remoteRepo, 'skos:inScheme\\:"' + taxonomy.shortId() + '"', undefined, undefined, {size: 10000})).map(x => x.shortId()); - } - EcRepository.alwaysTryUrl = true; + thing.id = thing.shortId(); + thing["schema:dateModified"] = new Date().toISOString(); + delete thing.loading; + delete thing.success; + delete thing.error; + delete thing.checked; + delete thing.directory; + }, + saveSubobjects(subObjects, toSave, callback) { this.precacheRemoteServer(subObjects, function() { new EcAsyncHelper().each(subObjects, function(subId, done) { EcRepository.get(subId, function(result) { @@ -706,42 +695,48 @@ export default { toSave.push(newObj); done(); }, done); - }, function(allDone) { - EcRepository.alwaysTryUrl = false; - me.repo.multiput(toSave, function() { - me.cassTaxonomies[firstIndex].loading = false; - me.cassTaxonomies[firstIndex].success = true; - me.$store.commit('editor/framework', taxonomy); - me.spitEvent("importFinished", taxonomy.shortId(), "importPage"); - me.importCassTaxonomies(); - }, function() { - me.cassTaxonomies[firstIndex].loading = false; - me.cassTaxonomies[firstIndex].error = true; - me.importCassTaxonomies(); - }); - }); + }, callback); }, function(error) { appError(error); }); }, + async continueCassTaxonomyImport(dataArray) { + var data = dataArray[0]; + var firstIndex = dataArray[1]; + var me = this; + let taxonomy = new EcConceptScheme(); + taxonomy.copyFrom(data); + this.cleanData(taxonomy); + // To do: address directory + let toSave = []; + toSave.push(taxonomy); + let subObjects = []; + if (taxonomy['skos:hasTopConcept'] && taxonomy['skos:hasTopConcept'].length) { + subObjects = (await EcConcept.search(this.remoteRepo, 'skos:inScheme\\:"' + taxonomy.shortId() + '"', undefined, undefined, {size: 10000})).map(x => x.shortId()); + } + EcRepository.alwaysTryUrl = true; + this.saveSubobjects(subObjects, toSave, function() { + EcRepository.alwaysTryUrl = false; + me.repo.multiput(toSave, function() { + me.cassTaxonomies[firstIndex].loading = false; + me.cassTaxonomies[firstIndex].success = true; + me.$store.commit('editor/framework', taxonomy); + me.spitEvent("importFinished", taxonomy.shortId(), "importPage"); + me.importCassTaxonomies(); + }, function() { + me.cassTaxonomies[firstIndex].loading = false; + me.cassTaxonomies[firstIndex].error = true; + me.importCassTaxonomies(); + }); + }); + }, continueCassFrameworkImport(dataArray) { var data = dataArray[0]; var firstIndex = dataArray[1]; var me = this; let framework = new EcFramework(); framework.copyFrom(data); - delete framework.owner; - delete framework.reader; - if (EcIdentityManager.default.ids.length > 0) { - framework.addOwner(EcIdentityManager.default.ids[0].ppk.toPk()); - } - framework.id = framework.shortId(); - framework["schema:dateModified"] = new Date().toISOString(); - delete framework.loading; - delete framework.success; - delete framework.error; - delete framework.checked; - delete framework.directory; + this.cleanData(framework); // To do: address directory let toSave = []; toSave.push(framework); @@ -756,40 +751,19 @@ export default { subObjects = subObjects.concat(framework.level); } EcRepository.alwaysTryUrl = true; - this.precacheRemoteServer(subObjects, function() { - new EcAsyncHelper().each(subObjects, function(subId, done) { - EcRepository.get(subId, function(result) { - let type = "Ec" + result.type; - if (type === "EcRelation") { - type = "EcAlignment"; - } - let newObj = new window[type](); - newObj.copyFrom(result); - delete newObj.owner; - delete newObj.reader; - if (EcIdentityManager.default.ids.length > 0) { - newObj.addOwner(EcIdentityManager.default.ids[0].ppk.toPk()); - } - newObj.id = newObj.shortId(); - toSave.push(newObj); - done(); - }, done); - }, function(allDone) { - EcRepository.alwaysTryUrl = false; - me.repo.multiput(toSave, function() { - me.cassFrameworks[firstIndex].loading = false; - me.cassFrameworks[firstIndex].success = true; - me.$store.commit('editor/framework', framework); - me.spitEvent("importFinished", framework.shortId(), "importPage"); - me.importCassFrameworks(); - }, function() { - me.cassFrameworks[firstIndex].loading = false; - me.cassFrameworks[firstIndex].error = true; - me.importCassFrameworks(); - }); + this.saveSubobjects(subObjects, toSave, function() { + EcRepository.alwaysTryUrl = false; + me.repo.multiput(toSave, function() { + me.cassFrameworks[firstIndex].loading = false; + me.cassFrameworks[firstIndex].success = true; + me.$store.commit('editor/framework', framework); + me.spitEvent("importFinished", framework.shortId(), "importPage"); + me.importCassFrameworks(); + }, function() { + me.cassFrameworks[firstIndex].loading = false; + me.cassFrameworks[firstIndex].error = true; + me.importCassFrameworks(); }); - }, function(error) { - appError(error); }); }, openDirectory: async function(directory) {