feat(Persona Imports): Allow homepage constants (#1408)

* csv import account ifi can now have it's id set as a constant. * Fixed changing the constant clearing the record. * Fixed deleting useConstant value. * Disabbled account key. * Update personasImportHelpers.js * Fixed tests. * Update personasImportHelpers.js * Update HeaderItem.spec.js.snap * Fixed account home page getting incorrectly set. * Will unset Account key. * Added migration. * Added down. * Remove unused import
LearningLocker · Jul 12, 2019 · 4a0f743 · 4a0f743
1 parent 244a15d
commit 4a0f743
Show file tree

Hide file tree

Showing 9 changed files with 267 additions and 53 deletions.
diff --git a/cli/src/commands/v2-migrations/20190711090000_persona_imports.js b/cli/src/commands/v2-migrations/20190711090000_persona_imports.js
@@ -0,0 +1,80 @@
+import { map as bmap } from 'bluebird';
+import { filter, get, forEach, set, toPairs, unset, cloneDeep } from 'lodash';
+import PersonasImports from 'lib/models/personasImport';
+import highland from 'highland';
+
+const BATCH_SIZE = 10000;
+
+const up = async () => {
+  const personasImportsSream = highland(PersonasImports.find({}).cursor());
+
+  const migrationStream = personasImportsSream.batch(BATCH_SIZE).flatMap((personasImports) => {
+    const donePromise = bmap(personasImports, async (personasImport) => {
+      const toMove = filter(toPairs(personasImport.structure), (structurePair) => {
+        const structure = structurePair[1];
+        return get(structure, 'columnType') === 'COLUMN_ACCOUNT_KEY';
+      });
+      if (toMove.length === 0) {
+        // Nothing to do
+        return;
+      }
+
+      const newStructure = cloneDeep(personasImport.structure);
+
+      forEach(toMove, (toMov) => {
+        const key = toMov[0];
+        const value = toMov[1];
+        set(newStructure, [get(value, 'relatedColumn'), 'primary'], get(value, 'primary'));
+        unset(newStructure, [key, 'primary']);
+      });
+
+      personasImport.structure = newStructure;
+      await personasImport.save();
+    }, { concurrency: 1 });
+
+    return highland(donePromise);
+  });
+
+  await new Promise((resolve, reject) => {
+    migrationStream.on('error', reject);
+    migrationStream.apply(resolve);
+  });
+};
+
+const down = async () => {
+  const personasImportsSream = highland(PersonasImports.find({}).cursor());
+
+  const migrationStream = personasImportsSream.batch(BATCH_SIZE).flatMap((personasImports) => {
+    const donePromise = bmap(personasImports, async (personasImport) => {
+      const toMove = filter(toPairs(personasImport.structure), (structurePair) => {
+        const structure = structurePair[1];
+        return get(structure, 'columnType') === 'COLUMN_ACCOUNT_VALUE';
+      });
+      if (toMove.length === 0) {
+        // Nothing to do
+        return;
+      }
+
+      const newStructure = cloneDeep(personasImport.structure);
+
+      forEach(toMove, (toMov) => {
+        const key = toMov[0];
+        const value = toMov[1];
+        set(newStructure, [get(value, 'relatedColumn'), 'primary'], get(value, 'primary'));
+        unset(newStructure, [key, 'primary']);
+      });
+
+      personasImport.structure = newStructure;
+      await personasImport.save();
+    }, { concurrency: 1 });
+
+    return highland(donePromise);
+  });
+
+  await new Promise((resolve, reject) => {
+    migrationStream.on('error', reject);
+    migrationStream.apply(resolve);
+  });
+};
+
+export default { up, down };
diff --git a/cli/src/commands/v2-migrations/index.js b/cli/src/commands/v2-migrations/index.js
@@ -10,6 +10,7 @@ import personaAttributesIndexes from './20190212150000_personaAttributes_indexes
 import oauthTokens from './20190131000000_oauth_tokens';
 import migrateToInQueries from './20190307130000_migrate_to_in_queries';
 import migrateResetTokens from './20190509000000_migrate_null_resetTokens';
+import personaImports from './20190711090000_persona_imports';
 
 export default new OrderedMap()
   .set('20171122100800_common_indexes', commonIndexesMigration)
@@ -22,4 +23,5 @@ export default new OrderedMap()
   .set('20190131000000_oauth_tokens', oauthTokens)
   .set('20190212150000_personaAttributes_indexes', personaAttributesIndexes)
   .set('20190307130000_migrate_to_in_queries', migrateToInQueries)
-  .set('20190509000000_migrate_null_resetTokens', migrateResetTokens);
+  .set('20190509000000_migrate_null_resetTokens', migrateResetTokens)
+  .set('20190711090000_persona_imports', personaImports);
diff --git a/lib/constants/personasImport.js b/lib/constants/personasImport.js
@@ -12,6 +12,7 @@ export const COLUMN_OPENID = 'COLUMN_OPENID';
 export const COLUMN_ACCOUNT_KEY = 'COLUMN_ACCOUNT_KEY';
 export const COLUMN_ACCOUNT_VALUE = 'COLUMN_ACCOUNT_VALUE';
 export const COLUMN_ATTRIBUTE_DATA = 'COLUMN_ATTRIBUTE_DATA';
+export const COLUMN_NOTHING = '';
 
 export const COLUMN_TYPES = [
   COLUMN_NAME,
@@ -52,7 +53,7 @@ export const IDENTIFIER_KEYS = [
   COLUMN_MBOX,
   COLUMN_MBOXSHA1SUM,
   COLUMN_OPENID,
-  COLUMN_ACCOUNT_KEY
+  COLUMN_ACCOUNT_VALUE
 ];
 
 // QUEUE

diff --git a/lib/models/personasImport.js b/lib/models/personasImport.js
@@ -9,7 +9,8 @@ import {
   includes,
   isUndefined,
   isInteger,
-  isNull
+  isNull,
+  get
 } from 'lodash';
 import * as scopes from 'lib/constants/scopes';
 import scopeChecks from 'lib/models/plugins/scopeChecks';
@@ -61,10 +62,17 @@ export const validateIsBound = (structure) => {
         columnType: value.columnType
       });
 
-      if (!value.relatedColumn && !includes(relatedColumns, '')) {
+      if (!value.relatedColumn && !includes(relatedColumns, '') && !get(value, 'useConstant', false)) {
         return [`${key} is not bound to another column`];
       }
 
+      if (
+        get(value, 'useConstant', false) &&
+        !get(value, 'constant', '').match(/^http(s?):\/\//)
+      ) {
+        return [`${key} constant should be a url`];
+      }
+
       if (!hasRelatedField(value.columnType) && value.relatedColumn) {
         return [`${key} is related to ${value.relatedColumn} but ${value.columnType} should have no relation`];
       }

diff --git a/lib/services/importPersonas/personasImportHelpers-test.js b/lib/services/importPersonas/personasImportHelpers-test.js
@@ -228,20 +228,27 @@ describe('personas import helper', () => {
         },
         test3: {
           columnType: COLUMN_ACCOUNT_KEY,
-          primary: 1,
           relatedColumn: 'test4'
         },
         test4: {
           columnType: COLUMN_ACCOUNT_VALUE,
+          primary: 1,
           relatedColumn: 'test3'
+        },
+        test5: {
+          columnType: COLUMN_ACCOUNT_VALUE,
+          primary: 3,
+          useConstant: true,
+          constant: 'http://hi'
         }
       };
 
       const row = {
         test1: 'dave',
         test2: '[email protected]',
         test3: 'http://dave.com',
-        test4: 'dave dot com'
+        test4: 'dave dot com',
+        test5: 'dave'
       };
 
       const result = getIfis({
@@ -256,7 +263,11 @@ describe('personas import helper', () => {
       expect(result[1].key).to.equal('mbox_sha1sum');
       expect(result[1].value).to.equal('[email protected]');
 
-      expect(result.length).to.equal(2);
+      expect(result[2].key).to.equal('account');
+      expect(result[2].value.name).to.equal('dave');
+      expect(result[2].value.homePage).to.equal('http://hi');
+
+      expect(result.length).to.equal(3);
     });
 
     it('should add mailto to mbox', () => {

diff --git a/lib/services/importPersonas/personasImportHelpers.js b/lib/services/importPersonas/personasImportHelpers.js
@@ -5,9 +5,11 @@ import {
   COLUMN_LAST_NAME,
   COLUMN_IFI_KEY,
   COLUMN_ACCOUNT_KEY,
+  COLUMN_ACCOUNT_VALUE,
   IDENTIFIER_KEYS,
   COLUMN_ATTRIBUTE_DATA,
-  COLUMN_MBOX
+  COLUMN_MBOX,
+  COLUMN_NOTHING,
 } from 'lib/constants/personasImport';
 import {
   includes,
@@ -16,7 +18,8 @@ import {
   mapValues,
   chain,
   findKey,
-  isNull
+  isNull,
+  first
 } from 'lodash';
 
 export const allRelated = chain(RELATED_COLUMN_MAP)
@@ -57,7 +60,8 @@ const getRelatedTypes = (columnType) => {
 
 export const getPossibleRelatedColumns = ({
   columnType,
-  structure // raw object
+  structure, // raw object
+  nothing = false
 }) => {
   const relatedTypes = getRelatedTypes(columnType);
 
@@ -69,7 +73,7 @@ export const getPossibleRelatedColumns = ({
   );
 
   const relatedStructures = filter(structureWithName, column =>
-    includes(relatedTypes, column.columnType)
+    includes(relatedTypes, column.columnType) || (nothing && includes(COLUMN_NOTHING, column.columnType))
   );
 
   const out = chain(relatedStructures)
@@ -85,6 +89,19 @@ export const getPossibleRelatedColumns = ({
   return out;
 };
 
+export const getPossibleRelatedAndNothingColumns = ({
+  columnType,
+  structure
+}) => {
+  const out = getPossibleRelatedColumns({
+    columnType,
+    structure,
+    nothing: true
+  });
+
+  return out;
+};
+
 export const resetRelatedStructure = ({
   structure, // immutablejs
   columnName
@@ -94,15 +111,19 @@ export const resetRelatedStructure = ({
       item.get('relatedColumn') === columnName ||
       key === columnName
     ) {
-      return item.set('relatedColumn', '');
+      let newItem = item;
+      if (item.get('columnType') === COLUMN_ACCOUNT_KEY) {
+        newItem = item.set('columnType', '');
+      }
+      return newItem.set('relatedColumn', '');
     }
     return item;
   });
 
 
 export const updateRelatedStructure = ({
   structure, // immutablejs
-  columnName,
+  columnName = '',
   relatedColumn
 }) => {
   const resetStructure = resetRelatedStructure({
@@ -118,7 +139,11 @@ export const updateRelatedStructure = ({
   const structureWithRelatedColumn = resetStructureRelated.setIn([columnName, 'relatedColumn'], relatedColumn);
 
   if (relatedColumn) {
-    const out = structureWithRelatedColumn.setIn([relatedColumn, 'relatedColumn'], columnName);
+    const out =
+      structureWithRelatedColumn
+        .setIn([relatedColumn, 'useConstant'], false)
+        .setIn([relatedColumn, 'columnType'], first(getRelatedTypes(structure.getIn([columnName, 'columnType']))))
+        .setIn([relatedColumn, 'relatedColumn'], columnName);
     return out;
   }
 
@@ -183,13 +208,17 @@ export const getIfis = ({
       })
     )
     .toArray()
-    .filter(({ columnType }) => includes(IDENTIFIER_KEYS, columnType))
+    .filter(columnStructure => isColumnOrderable({
+      columnStructure
+    }))
     .orderBy('primary');
 
   const ifis = ifiKeyColumns.map(({
     columnName,
     columnType,
-    relatedColumn
+    relatedColumn,
+    useConstant = false,
+    constant,
   }) => {
     if (columnType === COLUMN_IFI_KEY) {
       if ( // ignore un set rows in the csv
@@ -204,7 +233,7 @@ export const getIfis = ({
       };
     }
 
-    if (columnType === COLUMN_ACCOUNT_KEY) {
+    if (columnType === COLUMN_ACCOUNT_VALUE && useConstant === false) {
       if ( // ignore un set rows in the csv
         !row[columnName] ||
         !row[relatedColumn]
@@ -215,8 +244,21 @@ export const getIfis = ({
       return {
         key: 'account',
         value: {
-          homePage: row[columnName],
-          name: row[relatedColumn]
+          homePage: row[relatedColumn],
+          name: row[columnName]
+        }
+      };
+    }
+
+    if (columnType === COLUMN_ACCOUNT_VALUE && useConstant === true /* && [implied] && useConstant === true */) {
+      if (!row[columnName]) {
+        return null;
+      }
+      return {
+        key: 'account',
+        value: {
+          homePage: constant,
+          name: row[columnName]
         }
       };
     }