From efbfce93623c2012c7e3d18cbe326b9d60aa1027 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Berkefeld?= Date: Mon, 12 Apr 2021 16:46:51 +0200 Subject: [PATCH 1/8] #9 first draft at adding filter definitions --- lib/MetadataTypeDefinitions.js | 1 + lib/MetadataTypeInfo.js | 1 + lib/metadataTypes/Filter.js | 193 +++++++++++- lib/metadataTypes/FilterDefinition.js | 279 ++++++++++++++++++ .../definitions/Filter.definition.js | 70 ++--- .../FilterDefinition.definition.js | 113 +++++++ 6 files changed, 622 insertions(+), 35 deletions(-) create mode 100644 lib/metadataTypes/FilterDefinition.js create mode 100644 lib/metadataTypes/definitions/FilterDefinition.definition.js diff --git a/lib/MetadataTypeDefinitions.js b/lib/MetadataTypeDefinitions.js index d6b931e75..01a08aeb4 100644 --- a/lib/MetadataTypeDefinitions.js +++ b/lib/MetadataTypeDefinitions.js @@ -20,6 +20,7 @@ const MetadataTypeDefinitions = { eventDefinition: require('./metadataTypes/definitions/EventDefinition.definition'), fileTransfer: require('./metadataTypes/definitions/FileTransfer.definition'), filter: require('./metadataTypes/definitions/Filter.definition'), + filterDefinition: require('./metadataTypes/definitions/FilterDefinition.definition'), folder: require('./metadataTypes/definitions/Folder.definition'), ftpLocation: require('./metadataTypes/definitions/FtpLocation.definition'), importFile: require('./metadataTypes/definitions/ImportFile.definition'), diff --git a/lib/MetadataTypeInfo.js b/lib/MetadataTypeInfo.js index 594af4d4e..8e0d19ed2 100644 --- a/lib/MetadataTypeInfo.js +++ b/lib/MetadataTypeInfo.js @@ -20,6 +20,7 @@ const MetadataTypeInfo = { eventDefinition: require('./metadataTypes/EventDefinition'), fileTransfer: require('./metadataTypes/FileTransfer'), filter: require('./metadataTypes/Filter'), + filterDefinition: require('./metadataTypes/FilterDefinition'), folder: require('./metadataTypes/Folder'), ftpLocation: require('./metadataTypes/FtpLocation'), importFile: require('./metadataTypes/ImportFile'), diff --git a/lib/metadataTypes/Filter.js b/lib/metadataTypes/Filter.js index 658fa493d..b45000ff4 100644 --- a/lib/metadataTypes/Filter.js +++ b/lib/metadataTypes/Filter.js @@ -1,6 +1,32 @@ 'use strict'; +/** + * @typedef {Object} FilterItem + * @property {number} categoryId folder id + * @property {string} [createdDate] - + * @property {string} customerKey key + * @property {string} destinationObjectId DE/List ID + * @property {1|2|3|4} destinationTypeId 1:SubscriberList, 2:DataExtension, 3:GroupWizard, 4:BehavioralData + * @property {string} filterActivityId ? + * @property {string} filterDefinitionId ObjectID of filterDefinition + * @property {string} modifiedDate - + * @property {string} name name + * @property {string} sourceObjectId DE/List ID + * @property {1|2|3|4} sourceTypeId 1:SubscriberList, 2:DataExtension, 3:GroupWizard, 4:BehavioralData + * @property {number} statusId ? + * + * @typedef {Object.} FilterMap + */ + const MetadataType = require('./MetadataType'); +const Util = require('../util/util'); + +const dataTypes = { + 1: 'List', + 2: 'DataExtension', + 3: 'Group Wizard', + 4: 'Behavioral Data', +}; /** * Filter MetadataType @@ -13,11 +39,176 @@ class Filter extends MetadataType { * but only with some of the fields. So it is needed to loop over * Filters with the endpoint /automation/v1/filters/{id} * @param {String} retrieveDir Directory where retrieved metadata directory will be saved - * @returns {Promise} Promise + * @returns {Promise<{metadata:FilterMap,type:string}>} Promise of items */ static async retrieve(retrieveDir) { return super.retrieveREST(retrieveDir, '/automation/v1/filters/', null); } + /** + * manages post retrieve steps + * @param {FilterItem} item a single record + * @returns {FilterItem} parsed metadata definition + */ + static postRetrieveTasks(item) { + return this.parseMetadata(item); + } + /** + * parses retrieved Metadata before saving + * @param {FilterItem} metadata a single record + * @returns {FilterItem} parsed metadata definition + */ + static parseMetadata(metadata) { + try { + // folder + metadata.r__folder_Path = Util.getFromCache( + this.cache, + 'folder', + metadata.categoryId, + 'ID', + 'Path' + ); + delete metadata.categoryId; + + // filterDefinition + metadata.r__filterDefinition_CustomerKey = Util.getFromCache( + this.cache, + 'filterDefinition', + metadata.filterDefinitionId, + 'id', + 'key' + ); + delete metadata.filterDefinitionId; + + // source + if (metadata.sourceTypeId === 1) { + // list + } else if (metadata.sourceTypeId === 2) { + // dataExtension + metadata.r__source_dataExtension_CustomerKey = Util.getFromCache( + this.cache, + 'dataExtension', + metadata.sourceObjectId, + 'ObjectID', + 'CustomerKey' + ); + delete metadata.sourceObjectId; + delete metadata.sourceTypeId; + } else { + Util.logger.error( + `Filter '${metadata.name}' (${metadata.customerKey}): Unsupported source type ${ + metadata.sourceTypeId + }=${dataTypes[metadata.sourceTypeId]}` + ); + } + + // target + if (metadata.destinationTypeId === 1) { + // list + } else if (metadata.destinationTypeId === 2) { + // dataExtension + metadata.r__destination_dataExtension_CustomerKey = Util.getFromCache( + this.cache, + 'dataExtension', + metadata.destinationObjectId, + 'ObjectID', + 'CustomerKey' + ); + delete metadata.destinationObjectId; + delete metadata.destinationTypeId; + } else { + Util.logger.error( + `Filter '${metadata.name}' (${ + metadata.customerKey + }): Unsupported destination type ${metadata.destinationTypeId}=${ + dataTypes[metadata.destinationTypeId] + }` + ); + } + } catch (ex) { + Util.logger.error(`Filter '${metadata.name}' (${metadata.customerKey}): ${ex.message}`); + } + return metadata; + } + /** + * prepares a record for deployment + * @param {FilterItem} metadata a single record + * @returns {Promise} Promise of updated single record + */ + static async preDeployTasks(metadata) { + // folder + if (metadata.r__folder_Path) { + metadata.categoryId = Util.getFromCache( + this.cache, + 'folder', + metadata.r__folder_Path, + 'Path', + 'ID' + ); + delete metadata.r__folder_Path; + } + + // filterDefinition + if (metadata.r__filterDefinition_CustomerKey) { + metadata.filterDefinitionId = Util.getFromCache( + this.cache, + 'filterDefinition', + metadata.r__filterDefinition_CustomerKey, + 'CustomerKey', + 'ObjectID' + ); + delete metadata.r__filterDefinition_CustomerKey; + } + + // source + if (metadata.sourceTypeId === 1) { + // list + } else if (metadata.r__source_dataExtension_CustomerKey) { + // dataExtension + metadata.sourceObjectId = Util.getFromCache( + this.cache, + 'dataExtension', + metadata.r__source_dataExtension_CustomerKey, + 'CustomerKey', + 'ObjectID' + ); + metadata.sourceTypeId = 2; + delete metadata.r__source_dataExtension_CustomerKey; + } else { + // assume the type id is still in the metadata + throw new Error( + `Filter '${metadata.name}' (${metadata.customerKey}): Unsupported source type ${ + metadata.sourceTypeId + }=${dataTypes[metadata.sourceTypeId]}` + ); + } + + // target + if (metadata.destinationTypeId === 1) { + // list + } else if (metadata.r__destination_dataExtension_CustomerKey) { + // dataExtension + metadata.destinationObjectId = Util.getFromCache( + this.cache, + 'dataExtension', + metadata.r__destination_dataExtension_CustomerKey, + 'CustomerKey', + 'ObjectID' + ); + metadata.destinationTypeId = 2; + delete metadata.r__destination_dataExtension_CustomerKey; + } else { + // assume the type id is still in the metadata + throw new Error( + `Filter '${metadata.name}' (${ + metadata.customerKey + }): Unsupported destination type ${metadata.destinationTypeId}=${ + dataTypes[metadata.destinationTypeId] + }` + ); + } + + return metadata; + } } // Assign definition to static attributes diff --git a/lib/metadataTypes/FilterDefinition.js b/lib/metadataTypes/FilterDefinition.js new file mode 100644 index 000000000..2eeadd855 --- /dev/null +++ b/lib/metadataTypes/FilterDefinition.js @@ -0,0 +1,279 @@ +'use strict'; + +/** + * @typedef {Object} FilterDefinitionSOAPItem + * @property {string} ObjectID id + * @property {string} CustomerKey key + * @property {Object} [DataFilter] most relevant part that defines the filter + * @property {Object} DataFilter.LeftOperand - + * @property {string} DataFilter.LeftOperand.Property - + * @property {string} DataFilter.LeftOperand.SimpleOperator - + * @property {string} DataFilter.LeftOperand.Value - + * @property {string} DataFilter.LogicalOperator - + * @property {Object} [DataFilter.RightOperand] - + * @property {string} DataFilter.RightOperand.Property - + * @property {string} DataFilter.RightOperand.SimpleOperator - + * @property {string} DataFilter.RightOperand.Value - + * @property {string} Name name + * @property {string} Description - + * @property {string} [ObjectState] returned from SOAP API; used to return error messages + * + * @typedef {Object.} FilterDefinitionSOAPItemMap + + * + * /automation/v1/filterdefinitions/ (not used) + * @typedef {Object} AutomationFilterDefinitionItem + * @property {string} id object id + * @property {string} key external key + * @property {string} createdDate - + * @property {number} createdBy user id + * @property {string} createdName - + * @property {string} [description] (omitted by API if empty) + * @property {string} modifiedDate - + * @property {number} modifiedBy user id + * @property {string} modifiedName - + * @property {string} name name + * @property {string} categoryId folder id + * @property {string} filterDefinitionXml from REST API defines the filter in XML form + * @property {1|2} derivedFromType 1:list/profile attributes/measures, 2: dataExtension + * @property {boolean} isSendable ? + * @property {Object} [soap__DataFilter] copied from SOAP API, defines the filter in readable form + * @property {Object} soap__DataFilter.LeftOperand - + * @property {string} soap__DataFilter.LeftOperand.Property - + * @property {string} soap__DataFilter.LeftOperand.SimpleOperator - + * @property {string} soap__DataFilter.LeftOperand.Value - + * @property {string} soap__DataFilter.LogicalOperator - + * @property {Object} [soap__DataFilter.RightOperand] - + * @property {string} soap__DataFilter.RightOperand.Property - + * @property {string} soap__DataFilter.RightOperand.SimpleOperator - + * @property {string} soap__DataFilter.RightOperand.Value - + * + * /email/v1/filters/filterdefinition/ + * @typedef {Object} FilterDefinitionItem + * @property {string} id object id + * @property {string} key external key + * @property {string} createdDate date + * @property {number} createdBy user id + * @property {string} createdName name + * @property {string} [description] (omitted by API if empty) + * @property {string} lastUpdated date + * @property {number} lastUpdatedBy user id + * @property {string} lastUpdatedName name + * @property {string} name name + * @property {string} categoryId folder id + * @property {string} filterDefinitionXml from REST API defines the filter in XML form + * @property {1|2} derivedFromType 1:list/profile attributes/measures, 2: dataExtension + * @property {string} derivedFromObjectId Id of DataExtension - present if derivedFromType=2 + * @property {'DataExtension'|'SubscriberAttributes'} derivedFromObjectTypeName - + * @property {string} [derivedFromObjectName] name of DataExtension + * @property {boolean} isSendable ? + * @property {Object} [soap__DataFilter] copied from SOAP API, defines the filter in readable form + * @property {Object} soap__DataFilter.LeftOperand - + * @property {string} soap__DataFilter.LeftOperand.Property - + * @property {string} soap__DataFilter.LeftOperand.SimpleOperator - + * @property {string} soap__DataFilter.LeftOperand.Value - + * @property {string} soap__DataFilter.LogicalOperator - + * @property {Object} [soap__DataFilter.RightOperand] - + * @property {string} soap__DataFilter.RightOperand.Property - + * @property {string} soap__DataFilter.RightOperand.SimpleOperator - + * @property {string} soap__DataFilter.RightOperand.Value - + + * + * @typedef {Object.} FilterDefinitionMap + */ + +const MetadataType = require('./MetadataType'); +const Util = require('../util/util'); +const xml2js = require('xml2js'); + +/** + * FilterDefinition MetadataType + * @augments MetadataType + */ +class FilterDefinition extends MetadataType { + /** + * Retrieves all records and saves it to disk + * @param {string} retrieveDir Directory where retrieved metadata directory will be saved + * @returns {Promise<{metadata:FilterDefinitionMap,type:string}>} Promise of items + */ + static async retrieve(retrieveDir) { + // #1 get the list via SOAP cause the corresponding REST call has no BU filter apparently + // for reference the rest path: '/automation/v1/filterdefinitions?view=categoryinfo' + const keyFieldBak = this.definition.keyField; + const soapFields = ['DataFilter', 'ObjectID', 'CustomerKey', 'Description', 'Name']; + this.definition.keyField = 'CustomerKey'; + /** + * @type {FilterDefinitionSOAPItemMap[]} + */ + const responseObject = await this.retrieveSOAPBody(soapFields); + this.definition.keyField = keyFieldBak; + + // convert back to array + /** + * @type {FilterDefinitionSOAPItem[]} + */ + const listResponse = Object.keys(responseObject) + .map((key) => responseObject[key]) + .filter((item) => { + if (item.ObjectState) { + Util.logger.debug( + `Filtered filterDefinition ${item.name}: ${item.ObjectState}` + ); + return false; + } else { + return true; + } + }); + + // #2 + // /automation/v1/filterdefinitions/ + const response = ( + await Promise.all( + listResponse.map((item) => + this.client.RestClient.get({ + uri: '/email/v1/filters/filterdefinition/' + item.ObjectID, + }) + ) + ) + ) + .map((item) => item.body) + .map((item) => { + // description is not returned when empty + item.description = item.description || ''; + // add extra info from XML + item.c__soap_DataFilter = responseObject[item.key].DataFilter; + return item; + }); + const results = this.parseResponseBody({ Results: response }); + if (retrieveDir) { + const savedMetadata = await this.saveResults(results, retrieveDir, null, null); + Util.logger.info( + `Downloaded: ${this.definition.type} (${Object.keys(savedMetadata).length})` + ); + } + + return { metadata: results, type: this.definition.type }; + + // return super.retrieveSOAPgeneric(retrieveDir); + } + /** + * Retrieves all records for caching + * @returns {Promise<{metadata:FilterDefinitionMap,type:string}>} Promise of items + */ + static async retrieveForCache() { + return this.retrieve(null); + } + + /** + * manages post retrieve steps + * @param {FilterDefinitionItem} item a single record + * @returns {FilterDefinitionItem} parsed metadata definition + */ + static async postRetrieveTasks(item) { + return this.parseMetadata(item); + } + /** + * parses retrieved Metadata before saving + * @param {FilterDefinitionItem} metadata a single record + * @returns {FilterDefinitionItem} parsed metadata definition + */ + static async parseMetadata(metadata) { + try { + // folder + metadata.r__folder_Path = Util.getFromCache( + this.cache, + 'folder', + metadata.categoryId, + 'ID', + 'Path' + ); + delete metadata.categoryId; + + if (metadata.derivedFromType === 2) { + // DataExtension + metadata.r__dataExtension_CustomerKey = Util.getFromCache( + this.cache, + 'dataExtension', + metadata.derivedFromObjectId, + 'ObjectID', + 'CustomerKey' + ); + } + delete metadata.derivedFromObjectId; + delete metadata.derivedFromType; + metadata.c__filterDefinition = await xml2js.parseStringPromise( + metadata.filterDefinitionXml /* , options */ + ); + + // TODO check if Condition ID needs to be resolved or can be ignored + } catch (ex) { + Util.logger.error( + `FilterDefinition '${metadata.name}' (${metadata.key}): ${ex.message}` + ); + } + return metadata; + } + /** + * prepares a item for deployment + * @param {FilterDefinitionItem} metadata a single record + * @returns {Promise} Promise of updated single item + */ + static async preDeployTasks(metadata) { + // folder + metadata.categoryId = Util.getFromCache( + this.cache, + 'folder', + metadata.r__folder_Path, + 'Path', + 'ID' + ); + delete metadata.r__folder_Path; + + if (metadata.derivedFromObjectTypeName === 'SubscriberAttributes') { + // List + metadata.derivedFromType = 1; + metadata.derivedFromObjectId = '00000000-0000-0000-0000-000000000000'; + } else { + // DataExtension + metadata.derivedFromType = 2; + + if (metadata.r__dataExtension_CustomerKey) { + metadata.derivedFromObjectId = Util.getFromCache( + this.cache, + 'dataExtension', + metadata.r__dataExtension_CustomerKey, + 'CustomerKey', + 'ObjectID' + ); + delete metadata.r__dataExtension_CustomerKey; + } + } + delete metadata.c__filterDefinition; + delete metadata.c__soap_DataFilter; + + return metadata; + } + /** + * Creates a single item + * @param {FilterDefinitionItem} metadata a single item + * @returns {Promise} Promise + */ + static create(metadata) { + // TODO test the create + return super.createREST(metadata, '/email/v1/filters/filterdefinition/'); + } + /** + * Updates a single item + * @param {FilterDefinitionItem} metadata a single item + * @returns {Promise} Promise + */ + static update(metadata) { + // TODO test the update + // TODO figure out how to get the ID on the fly + return super.updateREST(metadata, '/email/v1/filters/filterdefinition/' + metadata.Id); + } +} +// Assign definition to static attributes +FilterDefinition.definition = require('../MetadataTypeDefinitions').filterDefinition; + +module.exports = FilterDefinition; diff --git a/lib/metadataTypes/definitions/Filter.definition.js b/lib/metadataTypes/definitions/Filter.definition.js index b9503571f..8f1686f6c 100644 --- a/lib/metadataTypes/definitions/Filter.definition.js +++ b/lib/metadataTypes/definitions/Filter.definition.js @@ -1,6 +1,6 @@ module.exports = { bodyIteratorField: 'items', - dependencies: [], + dependencies: ['filterDefinition', 'list', 'dataExtension', 'folder'], hasExtended: false, idField: 'id', keyField: 'customerKey', @@ -8,56 +8,58 @@ module.exports = { restPagination: true, type: 'filter', typeDescription: - 'BETA: Part of how filtered Data Extensions are created. Depends on type "FilterDefinitions".', - typeRetrieveByDefault: false, + 'Used in automations to filter lists and DEs. Depends on type "FilterDefinitions".', + typeRetrieveByDefault: true, typeName: 'Automation: Filter Activity', fields: { + // https://developer.salesforce.com/docs/atlas.en-us.noversion.mc-apis.meta/mc-apis/filteractivity.htm categoryId: { - isCreateable: false, - isUpdateable: false, + isCreateable: true, + isUpdateable: true, retrieving: true, - template: false, + template: true, }, createdDate: { isCreateable: false, isUpdateable: false, - retrieving: true, + retrieving: false, template: false, }, customerKey: { - isCreateable: null, - isUpdateable: false, + isCreateable: true, + isUpdateable: true, retrieving: true, - template: false, + template: true, }, description: { - isCreateable: false, - isUpdateable: false, + isCreateable: true, + isUpdateable: true, retrieving: true, - template: false, + template: true, }, destinationObjectId: { - isCreateable: false, - isUpdateable: false, + isCreateable: true, + isUpdateable: true, retrieving: true, - template: false, + template: true, }, destinationTypeId: { - isCreateable: false, - isUpdateable: false, + isCreateable: true, + isUpdateable: true, retrieving: true, - template: false, + template: true, }, filterActivityId: { isCreateable: null, isUpdateable: null, retrieving: true, + template: true, }, filterDefinitionId: { - isCreateable: false, - isUpdateable: false, + isCreateable: true, + isUpdateable: true, retrieving: true, - template: false, + template: true, }, modifiedDate: { isCreateable: false, @@ -66,28 +68,28 @@ module.exports = { template: false, }, name: { - isCreateable: null, - isUpdateable: false, + isCreateable: true, + isUpdateable: true, retrieving: true, - template: false, + template: true, }, sourceObjectId: { - isCreateable: false, - isUpdateable: false, + isCreateable: true, + isUpdateable: true, retrieving: true, - template: false, + template: true, }, sourceTypeId: { - isCreateable: false, - isUpdateable: false, + isCreateable: true, + isUpdateable: true, retrieving: true, - template: false, + template: true, }, statusId: { - isCreateable: false, - isUpdateable: false, + isCreateable: true, + isUpdateable: true, retrieving: true, - template: false, + template: true, }, }, }; diff --git a/lib/metadataTypes/definitions/FilterDefinition.definition.js b/lib/metadataTypes/definitions/FilterDefinition.definition.js new file mode 100644 index 000000000..77f1c62b7 --- /dev/null +++ b/lib/metadataTypes/definitions/FilterDefinition.definition.js @@ -0,0 +1,113 @@ +module.exports = { + bodyIteratorField: 'Results', + dependencies: ['folder', 'dataExtension'], + filter: {}, + hasExtended: false, + idField: 'id', + keyField: 'key', + nameField: 'name', + restPagination: false, + type: 'filterDefinition', + typeDescription: + 'Defines an audience based on specified rules. Used by Filter Activities and Filtered DEs.', + typeRetrieveByDefault: true, + typeName: 'Filter Definition', + fields: { + id: { + isCreateable: false, + isUpdateable: false, + retrieving: false, + template: false, + }, + key: { + isCreateable: true, + isUpdateable: true, + retrieving: true, + template: true, + }, + createdDate: { + isCreateable: false, + isUpdateable: false, + retrieving: false, + template: false, + }, + createdBy: { + isCreateable: false, + isUpdateable: false, + retrieving: false, + template: false, + }, + createdByName: { + isCreateable: false, + isUpdateable: false, + retrieving: false, + template: false, + }, + lastUpdated: { + isCreateable: false, + isUpdateable: false, + retrieving: true, + template: false, + }, + lastUpdatedBy: { + isCreateable: false, + isUpdateable: false, + retrieving: false, + template: false, + }, + lastUpdatedName: { + isCreateable: false, + isUpdateable: false, + retrieving: false, + template: false, + }, + name: { + isCreateable: true, + isUpdateable: true, + retrieving: true, + template: true, + }, + categoryId: { + isCreateable: true, + isUpdateable: true, + retrieving: true, + template: true, + }, + filterDefinitionXml: { + isCreateable: true, + isUpdateable: true, + retrieving: true, + template: true, + }, + derivedFromType: { + isCreateable: true, + isUpdateable: true, + retrieving: true, + template: true, + }, + derivedFromObjectId: { + isCreateable: true, + isUpdateable: true, + retrieving: true, + template: true, + }, + derivedFromObjectTypeName: { + isCreateable: true, + isUpdateable: true, + retrieving: true, + template: true, + }, + derivedFromObjectName: { + isCreateable: false, + isUpdateable: false, + retrieving: true, + template: true, + }, + isSendable: { + isCreateable: true, + isUpdateable: true, + retrieving: true, + template: true, + }, + }, +}; From d4a0e53cade7f52e09008cbcd708a785c2d35fa2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Berkefeld?= Date: Sun, 17 Apr 2022 09:58:52 +0200 Subject: [PATCH 2/8] #9: make branch ready for 4.0.0 release --- lib/metadataTypes/Filter.js | 31 ++++----- lib/metadataTypes/FilterDefinition.js | 94 +++++++++++++-------------- package-lock.json | 1 + package.json | 1 + 4 files changed, 58 insertions(+), 69 deletions(-) diff --git a/lib/metadataTypes/Filter.js b/lib/metadataTypes/Filter.js index b45000ff4..c26e25a19 100644 --- a/lib/metadataTypes/Filter.js +++ b/lib/metadataTypes/Filter.js @@ -20,6 +20,7 @@ const MetadataType = require('./MetadataType'); const Util = require('../util/util'); +const cache = require('../util/cache'); const dataTypes = { 1: 'List', @@ -60,8 +61,7 @@ class Filter extends MetadataType { static parseMetadata(metadata) { try { // folder - metadata.r__folder_Path = Util.getFromCache( - this.cache, + metadata.r__folder_Path = cache.searchForField( 'folder', metadata.categoryId, 'ID', @@ -70,8 +70,7 @@ class Filter extends MetadataType { delete metadata.categoryId; // filterDefinition - metadata.r__filterDefinition_CustomerKey = Util.getFromCache( - this.cache, + metadata.r__filterDefinition_CustomerKey = cache.searchForField( 'filterDefinition', metadata.filterDefinitionId, 'id', @@ -84,8 +83,7 @@ class Filter extends MetadataType { // list } else if (metadata.sourceTypeId === 2) { // dataExtension - metadata.r__source_dataExtension_CustomerKey = Util.getFromCache( - this.cache, + metadata.r__source_dataExtension_CustomerKey = cache.searchForField( 'dataExtension', metadata.sourceObjectId, 'ObjectID', @@ -94,7 +92,7 @@ class Filter extends MetadataType { delete metadata.sourceObjectId; delete metadata.sourceTypeId; } else { - Util.logger.error( + Util.logger.warn( `Filter '${metadata.name}' (${metadata.customerKey}): Unsupported source type ${ metadata.sourceTypeId }=${dataTypes[metadata.sourceTypeId]}` @@ -106,8 +104,7 @@ class Filter extends MetadataType { // list } else if (metadata.destinationTypeId === 2) { // dataExtension - metadata.r__destination_dataExtension_CustomerKey = Util.getFromCache( - this.cache, + metadata.r__destination_dataExtension_CustomerKey = cache.searchForField( 'dataExtension', metadata.destinationObjectId, 'ObjectID', @@ -116,7 +113,7 @@ class Filter extends MetadataType { delete metadata.destinationObjectId; delete metadata.destinationTypeId; } else { - Util.logger.error( + Util.logger.warn( `Filter '${metadata.name}' (${ metadata.customerKey }): Unsupported destination type ${metadata.destinationTypeId}=${ @@ -125,7 +122,7 @@ class Filter extends MetadataType { ); } } catch (ex) { - Util.logger.error(`Filter '${metadata.name}' (${metadata.customerKey}): ${ex.message}`); + Util.logger.warn(`Filter '${metadata.name}' (${metadata.customerKey}): ${ex.message}`); } return metadata; } @@ -137,8 +134,7 @@ class Filter extends MetadataType { static async preDeployTasks(metadata) { // folder if (metadata.r__folder_Path) { - metadata.categoryId = Util.getFromCache( - this.cache, + metadata.categoryId = cache.searchForField( 'folder', metadata.r__folder_Path, 'Path', @@ -149,8 +145,7 @@ class Filter extends MetadataType { // filterDefinition if (metadata.r__filterDefinition_CustomerKey) { - metadata.filterDefinitionId = Util.getFromCache( - this.cache, + metadata.filterDefinitionId = cache.searchForField( 'filterDefinition', metadata.r__filterDefinition_CustomerKey, 'CustomerKey', @@ -164,8 +159,7 @@ class Filter extends MetadataType { // list } else if (metadata.r__source_dataExtension_CustomerKey) { // dataExtension - metadata.sourceObjectId = Util.getFromCache( - this.cache, + metadata.sourceObjectId = cache.searchForField( 'dataExtension', metadata.r__source_dataExtension_CustomerKey, 'CustomerKey', @@ -187,8 +181,7 @@ class Filter extends MetadataType { // list } else if (metadata.r__destination_dataExtension_CustomerKey) { // dataExtension - metadata.destinationObjectId = Util.getFromCache( - this.cache, + metadata.destinationObjectId = cache.searchForField( 'dataExtension', metadata.r__destination_dataExtension_CustomerKey, 'CustomerKey', diff --git a/lib/metadataTypes/FilterDefinition.js b/lib/metadataTypes/FilterDefinition.js index 2eeadd855..51abe9516 100644 --- a/lib/metadataTypes/FilterDefinition.js +++ b/lib/metadataTypes/FilterDefinition.js @@ -84,7 +84,8 @@ const MetadataType = require('./MetadataType'); const Util = require('../util/util'); -const xml2js = require('xml2js'); +const cache = require('../util/cache'); +const { XMLBuilder, XMLParser } = require('fast-xml-parser'); /** * FilterDefinition MetadataType @@ -99,52 +100,51 @@ class FilterDefinition extends MetadataType { static async retrieve(retrieveDir) { // #1 get the list via SOAP cause the corresponding REST call has no BU filter apparently // for reference the rest path: '/automation/v1/filterdefinitions?view=categoryinfo' - const keyFieldBak = this.definition.keyField; + const soapFields = ['DataFilter', 'ObjectID', 'CustomerKey', 'Description', 'Name']; - this.definition.keyField = 'CustomerKey'; /** * @type {FilterDefinitionSOAPItemMap[]} */ - const responseObject = await this.retrieveSOAPBody(soapFields); + const responseSOAP = await this.client.soap.retrieveBulk(this.definition.type, soapFields); + console.log('responseSOAP', responseSOAP); + + // backup REST value of the keyField + const keyFieldBak = this.definition.keyField; + this.definition.keyField = 'CustomerKey'; + const responseSOAPMap = this.parseResponseBody(responseSOAP); + // restore the keyField to its REST value this.definition.keyField = keyFieldBak; + console.log('responseSOAPMap', responseSOAPMap); - // convert back to array /** * @type {FilterDefinitionSOAPItem[]} */ - const listResponse = Object.keys(responseObject) - .map((key) => responseObject[key]) - .filter((item) => { - if (item.ObjectState) { - Util.logger.debug( - `Filtered filterDefinition ${item.name}: ${item.ObjectState}` - ); - return false; - } else { - return true; - } - }); + const responseSOAPList = responseSOAP.Results.filter((item) => { + if (item.ObjectState) { + Util.logger.debug(`Filtered filterDefinition ${item.name}: ${item.ObjectState}`); + return false; + } else { + return true; + } + }); // #2 // /automation/v1/filterdefinitions/ - const response = ( + const responseREST = ( await Promise.all( - listResponse.map((item) => - this.client.RestClient.get({ - uri: '/email/v1/filters/filterdefinition/' + item.ObjectID, - }) + responseSOAPList.map((item) => + this.client.rest.get('/email/v1/filters/filterdefinition/' + item.ObjectID) ) ) - ) - .map((item) => item.body) - .map((item) => { - // description is not returned when empty - item.description = item.description || ''; - // add extra info from XML - item.c__soap_DataFilter = responseObject[item.key].DataFilter; - return item; - }); - const results = this.parseResponseBody({ Results: response }); + ).map((item) => { + // description is not returned when empty + item.description = item.description || ''; + // add extra info from XML + item.c__soap_DataFilter = responseSOAPMap[item.key].DataFilter; + return item; + }); + console.log('responseREST', responseREST); + const results = this.parseResponseBody({ Results: responseREST }); if (retrieveDir) { const savedMetadata = await this.saveResults(results, retrieveDir, null, null); Util.logger.info( @@ -153,8 +153,6 @@ class FilterDefinition extends MetadataType { } return { metadata: results, type: this.definition.type }; - - // return super.retrieveSOAPgeneric(retrieveDir); } /** * Retrieves all records for caching @@ -180,8 +178,7 @@ class FilterDefinition extends MetadataType { static async parseMetadata(metadata) { try { // folder - metadata.r__folder_Path = Util.getFromCache( - this.cache, + metadata.r__folder_Path = cache.searchForField( 'folder', metadata.categoryId, 'ID', @@ -191,19 +188,20 @@ class FilterDefinition extends MetadataType { if (metadata.derivedFromType === 2) { // DataExtension - metadata.r__dataExtension_CustomerKey = Util.getFromCache( - this.cache, + metadata.r__dataExtension_CustomerKey = cache.searchForField( 'dataExtension', metadata.derivedFromObjectId, 'ObjectID', 'CustomerKey' ); } + metadata.del__derivedFromObjectId = metadata.derivedFromObjectId; // TEMP for DEBUGGING / remove before release + metadata.del__derivedFromType = metadata.derivedFromType; // TEMP for DEBUGGING / remove before release delete metadata.derivedFromObjectId; delete metadata.derivedFromType; - metadata.c__filterDefinition = await xml2js.parseStringPromise( - metadata.filterDefinitionXml /* , options */ - ); + + const xmlToJson = new XMLParser({ ignoreAttributes: false }); + metadata.c__filterDefinition = xmlToJson.parse(metadata.filterDefinitionXml); // TODO check if Condition ID needs to be resolved or can be ignored } catch (ex) { @@ -220,13 +218,7 @@ class FilterDefinition extends MetadataType { */ static async preDeployTasks(metadata) { // folder - metadata.categoryId = Util.getFromCache( - this.cache, - 'folder', - metadata.r__folder_Path, - 'Path', - 'ID' - ); + metadata.categoryId = cache.searchForField('folder', metadata.r__folder_Path, 'Path', 'ID'); delete metadata.r__folder_Path; if (metadata.derivedFromObjectTypeName === 'SubscriberAttributes') { @@ -238,8 +230,7 @@ class FilterDefinition extends MetadataType { metadata.derivedFromType = 2; if (metadata.r__dataExtension_CustomerKey) { - metadata.derivedFromObjectId = Util.getFromCache( - this.cache, + metadata.derivedFromObjectId = cache.searchForField( 'dataExtension', metadata.r__dataExtension_CustomerKey, 'CustomerKey', @@ -248,6 +239,9 @@ class FilterDefinition extends MetadataType { delete metadata.r__dataExtension_CustomerKey; } } + + const jsonToXml = new XMLBuilder({ ignoreAttributes: false }); + metadata.filterDefinitionXml = jsonToXml.build(metadata.c__filterDefinition); delete metadata.c__filterDefinition; delete metadata.c__soap_DataFilter; diff --git a/package-lock.json b/package-lock.json index b7d320163..5bfecead4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,7 @@ "command-exists": "1.2.9", "conf": "10.1.1", "console.table": "0.10.0", + "fast-xml-parser": "4.0.7", "fs-extra": "10.0.1", "inquirer": "8.2.2", "json-to-table": "4.2.1", diff --git a/package.json b/package.json index 9cce2236d..5f57cba68 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,7 @@ "command-exists": "1.2.9", "conf": "10.1.1", "console.table": "0.10.0", + "fast-xml-parser": "4.0.7", "fs-extra": "10.0.1", "inquirer": "8.2.2", "json-to-table": "4.2.1", From 0e519dba94f846fb624d4d4cbcc4d218511d4aef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Berkefeld?= Date: Tue, 29 Aug 2023 10:14:06 +0200 Subject: [PATCH 3/8] #9: ran lint:fix --- lib/metadataTypes/FilterDefinition.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/metadataTypes/FilterDefinition.js b/lib/metadataTypes/FilterDefinition.js index 12840fe24..979a96baa 100644 --- a/lib/metadataTypes/FilterDefinition.js +++ b/lib/metadataTypes/FilterDefinition.js @@ -59,7 +59,7 @@ class FilterDefinition extends MetadataType { ) ).map((item) => { // description is not returned when empty - item.description = item.description || ''; + item.description ||= ''; // add extra info from XML item.c__soap_DataFilter = responseSOAPMap[item.key].DataFilter; return item; From a742713496a8e6858fb12284608737a2748fc5b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Berkefeld?= Date: Tue, 29 Aug 2023 15:08:18 +0200 Subject: [PATCH 4/8] #9: improve retrieve filterDefinition --- docs/dist/documentation.md | 24 +-- lib/metadataTypes/FilterDefinition.js | 140 +++++++++++------- .../FilterDefinition.definition.js | 72 ++++++--- 3 files changed, 140 insertions(+), 96 deletions(-) diff --git a/docs/dist/documentation.md b/docs/dist/documentation.md index 1877f430a..319125d8c 100644 --- a/docs/dist/documentation.md +++ b/docs/dist/documentation.md @@ -2951,17 +2951,16 @@ FilterDefinition MetadataType **Extends**: [MetadataType](#MetadataType) * [FilterDefinition](#FilterDefinition) ⇐ [MetadataType](#MetadataType) - * [.retrieve(retrieveDir)](#FilterDefinition.retrieve) ⇒ Promise.<{metadata: TYPE.FilterDefinitionMap, type: string}> + * [.retrieve(retrieveDir, [_], [__], [key])](#FilterDefinition.retrieve) ⇒ Promise.<{metadata: TYPE.FilterDefinitionMap, type: string}> * [.retrieveForCache()](#FilterDefinition.retrieveForCache) ⇒ Promise.<{metadata: TYPE.FilterDefinitionMap, type: string}> - * [.postRetrieveTasks(item)](#FilterDefinition.postRetrieveTasks) ⇒ TYPE.FilterDefinitionItem - * [.parseMetadata(metadata)](#FilterDefinition.parseMetadata) ⇒ TYPE.FilterDefinitionItem + * [.postRetrieveTasks(metadata)](#FilterDefinition.postRetrieveTasks) ⇒ TYPE.FilterDefinitionItem * [.preDeployTasks(metadata)](#FilterDefinition.preDeployTasks) ⇒ Promise.<TYPE.FilterDefinitionItem> * [.create(metadata)](#FilterDefinition.create) ⇒ Promise.<TYPE.FilterDefinitionItem> * [.update(metadata)](#FilterDefinition.update) ⇒ Promise.<TYPE.FilterDefinitionItem> -### FilterDefinition.retrieve(retrieveDir) ⇒ Promise.<{metadata: TYPE.FilterDefinitionMap, type: string}> +### FilterDefinition.retrieve(retrieveDir, [_], [__], [key]) ⇒ Promise.<{metadata: TYPE.FilterDefinitionMap, type: string}> Retrieves all records and saves it to disk **Kind**: static method of [FilterDefinition](#FilterDefinition) @@ -2970,6 +2969,9 @@ Retrieves all records and saves it to disk | Param | Type | Description | | --- | --- | --- | | retrieveDir | string | Directory where retrieved metadata directory will be saved | +| [_] | void | unused parameter | +| [__] | void | unused parameter | +| [key] | string | customer key of single item to retrieve | @@ -2980,19 +2982,7 @@ Retrieves all records for caching **Returns**: Promise.<{metadata: TYPE.FilterDefinitionMap, type: string}> - Promise of items -### FilterDefinition.postRetrieveTasks(item) ⇒ TYPE.FilterDefinitionItem -manages post retrieve steps - -**Kind**: static method of [FilterDefinition](#FilterDefinition) -**Returns**: TYPE.FilterDefinitionItem - parsed metadata definition - -| Param | Type | Description | -| --- | --- | --- | -| item | TYPE.FilterDefinitionItem | a single record | - - - -### FilterDefinition.parseMetadata(metadata) ⇒ TYPE.FilterDefinitionItem +### FilterDefinition.postRetrieveTasks(metadata) ⇒ TYPE.FilterDefinitionItem parses retrieved Metadata before saving **Kind**: static method of [FilterDefinition](#FilterDefinition) diff --git a/lib/metadataTypes/FilterDefinition.js b/lib/metadataTypes/FilterDefinition.js index 979a96baa..5422e72eb 100644 --- a/lib/metadataTypes/FilterDefinition.js +++ b/lib/metadataTypes/FilterDefinition.js @@ -16,33 +16,49 @@ class FilterDefinition extends MetadataType { * Retrieves all records and saves it to disk * * @param {string} retrieveDir Directory where retrieved metadata directory will be saved + * @param {void} [_] unused parameter + * @param {void} [__] unused parameter + * @param {string} [key] customer key of single item to retrieve * @returns {Promise.<{metadata: TYPE.FilterDefinitionMap, type: string}>} Promise of items */ - static async retrieve(retrieveDir) { + static async retrieve(retrieveDir, _, __, key) { // #1 get the list via SOAP cause the corresponding REST call has no BU filter apparently // for reference the rest path: '/automation/v1/filterdefinitions?view=categoryinfo' - const soapFields = ['DataFilter', 'ObjectID', 'CustomerKey', 'Description', 'Name']; + const soapFields = ['DataFilter', 'ObjectID', 'CustomerKey', 'Name']; + let requestParams; + if (key) { + requestParams = { + filter: { + leftOperand: 'CustomerKey', + operator: 'equals', + rightOperand: key, + }, + }; + } + /** * @type {TYPE.FilterDefinitionSOAPItemMap[]} */ - const responseSOAP = await this.client.soap.retrieveBulk(this.definition.type, soapFields); - console.log('responseSOAP', responseSOAP); // eslint-disable-line no-console + const responseSOAP = await this.client.soap.retrieveBulk( + this.definition.type, + soapFields, + requestParams + ); // backup REST value of the keyField const keyFieldBak = this.definition.keyField; this.definition.keyField = 'CustomerKey'; - const responseSOAPMap = this.parseResponseBody(responseSOAP); + const responseSOAPMap = this.parseResponseBody(responseSOAP, key); // restore the keyField to its REST value this.definition.keyField = keyFieldBak; - console.log('responseSOAPMap', responseSOAPMap); // eslint-disable-line no-console /** * @type {TYPE.FilterDefinitionSOAPItem[]} */ const responseSOAPList = responseSOAP.Results.filter((item) => { if (item.ObjectState) { - Util.logger.debug(`Filtered filterDefinition ${item.name}: ${item.ObjectState}`); + Util.logger.debug(`Filtered filterDefinition ${item.Name}: ${item.ObjectState}`); return false; } else { return true; @@ -51,29 +67,28 @@ class FilterDefinition extends MetadataType { // #2 // /automation/v1/filterdefinitions/ - const responseREST = ( - await Promise.all( - responseSOAPList.map((item) => - this.client.rest.get('/email/v1/filters/filterdefinition/' + item.ObjectID) - ) + const metadataMap = ( + await super.retrieveRESTcollection( + responseSOAPList.map((item) => ({ + id: item.ObjectID, + uri: '/email/v1/filters/filterdefinition/' + item.ObjectID, + })) ) - ).map((item) => { + ).metadata; + for (const item of Object.values(metadataMap)) { // description is not returned when empty item.description ||= ''; // add extra info from XML item.c__soap_DataFilter = responseSOAPMap[item.key].DataFilter; - return item; - }); - console.log('responseREST', responseREST); // eslint-disable-line no-console - const results = this.parseResponseBody({ Results: responseREST }); + } if (retrieveDir) { - const savedMetadata = await this.saveResults(results, retrieveDir, null, null); + const savedMetadata = await this.saveResults(metadataMap, retrieveDir); Util.logger.info( `Downloaded: ${this.definition.type} (${Object.keys(savedMetadata).length})` ); } - return { metadata: results, type: this.definition.type }; + return { metadata: metadataMap, type: this.definition.type }; } /** * Retrieves all records for caching @@ -84,50 +99,62 @@ class FilterDefinition extends MetadataType { return this.retrieve(null); } - /** - * manages post retrieve steps - * - * @param {TYPE.FilterDefinitionItem} item a single record - * @returns {TYPE.FilterDefinitionItem} parsed metadata definition - */ - static async postRetrieveTasks(item) { - return this.parseMetadata(item); - } /** * parses retrieved Metadata before saving * * @param {TYPE.FilterDefinitionItem} metadata a single record * @returns {TYPE.FilterDefinitionItem} parsed metadata definition */ - static async parseMetadata(metadata) { + static async postRetrieveTasks(metadata) { + if (metadata.derivedFromType > 4) { + // GUI only shows types 1,2,3,4; lets mimic that here. + // type 6 seems to be journey related. Maybe we need to change that again in the future + return; + } try { // folder - metadata.r__folder_Path = cache.searchForField( - 'folder', - metadata.categoryId, - 'ID', - 'Path' - ); - delete metadata.categoryId; - - if (metadata.derivedFromType === 2) { - // DataExtension - metadata.r__dataExtension_CustomerKey = cache.searchForField( - 'dataExtension', - metadata.derivedFromObjectId, - 'ObjectID', - 'CustomerKey' - ); + this.setFolderPath(metadata); + + switch (metadata.derivedFromType) { + case 1: { + // SubscriberAttributes + // TODO + break; + } + case 2: { + // DataExtension + metadata.r__dataExtension_CustomerKey = cache.searchForField( + 'dataExtension', + metadata.derivedFromObjectId, + 'ObjectID', + 'CustomerKey' + ); + delete metadata.derivedFromObjectId; + delete metadata.derivedFromType; + break; + } + case 3: { + // TODO + break; + } + case 4: { + // TODO + break; + } + case 5: { + // TODO + break; + } + case 6: { + // TODO + break; + } } - metadata.del__derivedFromObjectId = metadata.derivedFromObjectId; // TEMP for DEBUGGING / remove before release - metadata.del__derivedFromType = metadata.derivedFromType; // TEMP for DEBUGGING / remove before release - delete metadata.derivedFromObjectId; - delete metadata.derivedFromType; const xmlToJson = new XMLParser({ ignoreAttributes: false }); metadata.c__filterDefinition = xmlToJson.parse(metadata.filterDefinitionXml); - - // TODO check if Condition ID needs to be resolved or can be ignored + // TODO map Condition ID to DataExtensionField ID + delete metadata.filterDefinitionXml; } catch (ex) { Util.logger.error( `FilterDefinition '${metadata.name}' (${metadata.key}): ${ex.message}` @@ -143,11 +170,10 @@ class FilterDefinition extends MetadataType { */ static async preDeployTasks(metadata) { // folder - metadata.categoryId = cache.searchForField('folder', metadata.r__folder_Path, 'Path', 'ID'); - delete metadata.r__folder_Path; + super.setFolderId(metadata); if (metadata.derivedFromObjectTypeName === 'SubscriberAttributes') { - // List + // SubscriberAttributes metadata.derivedFromType = 1; metadata.derivedFromObjectId = '00000000-0000-0000-0000-000000000000'; } else { @@ -190,8 +216,10 @@ class FilterDefinition extends MetadataType { */ static update(metadata) { // TODO test the update - // TODO figure out how to get the ID on the fly - return super.updateREST(metadata, '/email/v1/filters/filterdefinition/' + metadata.Id); + return super.updateREST( + metadata, + '/email/v1/filters/filterdefinition/' + metadata[this.definition.idField] + ); } } // Assign definition to static attributes diff --git a/lib/metadataTypes/definitions/FilterDefinition.definition.js b/lib/metadataTypes/definitions/FilterDefinition.definition.js index 77f1c62b7..83f51d78a 100644 --- a/lib/metadataTypes/definitions/FilterDefinition.definition.js +++ b/lib/metadataTypes/definitions/FilterDefinition.definition.js @@ -1,11 +1,17 @@ module.exports = { bodyIteratorField: 'Results', - dependencies: ['folder', 'dataExtension'], + dependencies: ['folder-filterdefinition', 'folder-hidden', 'dataExtension'], filter: {}, hasExtended: false, idField: 'id', keyField: 'key', nameField: 'name', + folderType: 'filterdefinition', + folderIdField: 'categoryId', + createdDateField: 'createdDate', + createdNameField: 'createdBy', + lastmodDateField: 'lastUpdated', + lastmodNameField: 'lastUpdatedBy', restPagination: false, type: 'filterDefinition', typeDescription: @@ -13,9 +19,10 @@ module.exports = { typeRetrieveByDefault: true, typeName: 'Filter Definition', fields: { + // the GUI seems to ONLY send fields during update that are actually changed. It has yet to be tested if that also works with sending other fiedls as well id: { isCreateable: false, - isUpdateable: false, + isUpdateable: false, // included in URL retrieving: false, template: false, }, @@ -37,30 +44,30 @@ module.exports = { retrieving: false, template: false, }, - createdByName: { - isCreateable: false, - isUpdateable: false, - retrieving: false, - template: false, - }, + // createdByName: { + // isCreateable: false, + // isUpdateable: false, + // retrieving: false, + // template: false, + // }, lastUpdated: { - isCreateable: false, - isUpdateable: false, - retrieving: true, - template: false, - }, - lastUpdatedBy: { isCreateable: false, isUpdateable: false, retrieving: false, template: false, }, - lastUpdatedName: { + lastUpdatedBy: { isCreateable: false, isUpdateable: false, retrieving: false, template: false, }, + // lastUpdatedName: { + // isCreateable: false, + // isUpdateable: false, + // retrieving: false, + // template: false, + // }, name: { isCreateable: true, isUpdateable: true, @@ -68,44 +75,63 @@ module.exports = { template: true, }, categoryId: { + // returned by GET / CREATE / UPDATE; used in CREATE payload isCreateable: true, isUpdateable: true, retrieving: true, template: true, }, + // CategoryId: { + // // used by UPDATE payload + // isCreateable: false, + // isUpdateable: true, + // retrieving: false, + // template: false, + // }, filterDefinitionXml: { isCreateable: true, isUpdateable: true, retrieving: true, template: true, }, + // DerivedFromType: { + // // this upper-cased spelling is used by GUI when creating a dataExtension based filterDefintion + // isCreateable: true, + // isUpdateable: false, // cannot be updated + // retrieving: false, + // template: false, + // }, derivedFromType: { + // 1: SubscriberAttributes, 2: DataExtension, 6: EntryCriteria; isCreateable: true, - isUpdateable: true, + isUpdateable: false, // cannot be updated retrieving: true, template: true, }, derivedFromObjectId: { + // dataExtension ID or '00000000-0000-0000-0000-000000000000' for lists isCreateable: true, - isUpdateable: true, + isUpdateable: false, // cannot be updated retrieving: true, template: true, }, derivedFromObjectTypeName: { - isCreateable: true, - isUpdateable: true, + // "SubscriberAttributes" | "DataExtension" | "EntryCriteria" ...; only returned by GET API + isCreateable: false, + isUpdateable: false, retrieving: true, template: true, }, derivedFromObjectName: { + // dataExtension name; field only returned by GET-API isCreateable: false, isUpdateable: false, - retrieving: true, - template: true, + retrieving: false, + template: false, }, isSendable: { - isCreateable: true, - isUpdateable: true, + isCreateable: false, // automatically set during create + isUpdateable: false, retrieving: true, template: true, }, From 15548a68a45f6117477f403405177bf0d1f88a4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Berkefeld?= Date: Wed, 30 Aug 2023 16:02:10 +0200 Subject: [PATCH 5/8] #9: switch to folder-based collection retrieve; resolve all filterDefinition details; split off hidden ones, resolve filterDefinition in filter --- docs/dist/documentation.md | 108 +++- lib/MetadataTypeDefinitions.js | 1 + lib/MetadataTypeInfo.js | 1 + lib/metadataTypes/DataExtension.js | 2 +- lib/metadataTypes/Filter.js | 50 +- lib/metadataTypes/FilterDefinition.js | 493 ++++++++++++++---- lib/metadataTypes/FilterDefinitionHidden.js | 24 + .../definitions/Filter.definition.js | 19 +- .../FilterDefinition.definition.js | 66 ++- .../FilterDefinitionHidden.definition.js | 156 ++++++ 10 files changed, 751 insertions(+), 169 deletions(-) create mode 100644 lib/metadataTypes/FilterDefinitionHidden.js create mode 100644 lib/metadataTypes/definitions/FilterDefinitionHidden.definition.js diff --git a/docs/dist/documentation.md b/docs/dist/documentation.md index 319125d8c..0df8883e8 100644 --- a/docs/dist/documentation.md +++ b/docs/dist/documentation.md @@ -70,6 +70,9 @@ as this is a configuration in the EID

FilterDefinition ⇐ MetadataType

FilterDefinition MetadataType

+
FilterDefinitionHidden ⇐ FilterDefinitionHidden
+

FilterDefinitionHidden MetadataType

+
Folder ⇐ MetadataType

Folder MetadataType

@@ -2884,8 +2887,7 @@ Filter MetadataType * [Filter](#Filter) ⇐ [MetadataType](#MetadataType) * [.retrieve(retrieveDir, [_], [__], [key])](#Filter.retrieve) ⇒ Promise.<{metadata: TYPE.FilterMap, type: string}> - * [.postRetrieveTasks(item)](#Filter.postRetrieveTasks) ⇒ TYPE.FilterItem - * [.parseMetadata(metadata)](#Filter.parseMetadata) ⇒ TYPE.FilterItem + * [.postRetrieveTasks(metadata)](#Filter.postRetrieveTasks) ⇒ TYPE.FilterItem * [.preDeployTasks(metadata)](#Filter.preDeployTasks) ⇒ Promise.<TYPE.FilterItem> @@ -2908,19 +2910,7 @@ Filters with the endpoint /automation/v1/filters/{id} -### Filter.postRetrieveTasks(item) ⇒ TYPE.FilterItem -manages post retrieve steps - -**Kind**: static method of [Filter](#Filter) -**Returns**: TYPE.FilterItem - parsed metadata definition - -| Param | Type | Description | -| --- | --- | --- | -| item | TYPE.FilterItem | a single record | - - - -### Filter.parseMetadata(metadata) ⇒ TYPE.FilterItem +### Filter.postRetrieveTasks(metadata) ⇒ TYPE.FilterItem parses retrieved Metadata before saving **Kind**: static method of [Filter](#Filter) @@ -2952,8 +2942,15 @@ FilterDefinition MetadataType * [FilterDefinition](#FilterDefinition) ⇐ [MetadataType](#MetadataType) * [.retrieve(retrieveDir, [_], [__], [key])](#FilterDefinition.retrieve) ⇒ Promise.<{metadata: TYPE.FilterDefinitionMap, type: string}> + * [.getFilterFolderIds([hidden])](#FilterDefinition.getFilterFolderIds) ⇒ Array.<number> + * [.getMeasureFolderIds()](#FilterDefinition.getMeasureFolderIds) ⇒ Array.<number> + * [.cacheDeFields(metadataTypeMapObj)](#FilterDefinition.cacheDeFields) + * [.cacheContactAttributes(metadataTypeMapObj)](#FilterDefinition.cacheContactAttributes) + * [.cacheMeasures(metadataTypeMapObj)](#FilterDefinition.cacheMeasures) * [.retrieveForCache()](#FilterDefinition.retrieveForCache) ⇒ Promise.<{metadata: TYPE.FilterDefinitionMap, type: string}> * [.postRetrieveTasks(metadata)](#FilterDefinition.postRetrieveTasks) ⇒ TYPE.FilterDefinitionItem + * [.resolveFieldIds(metadata, [fieldCache], [filter])](#FilterDefinition.resolveFieldIds) ⇒ void + * [.resolveAttributeIds(metadata, [filter])](#FilterDefinition.resolveAttributeIds) ⇒ void * [.preDeployTasks(metadata)](#FilterDefinition.preDeployTasks) ⇒ Promise.<TYPE.FilterDefinitionItem> * [.create(metadata)](#FilterDefinition.create) ⇒ Promise.<TYPE.FilterDefinitionItem> * [.update(metadata)](#FilterDefinition.update) ⇒ Promise.<TYPE.FilterDefinitionItem> @@ -2973,6 +2970,52 @@ Retrieves all records and saves it to disk | [__] | void | unused parameter | | [key] | string | customer key of single item to retrieve | + + +### FilterDefinition.getFilterFolderIds([hidden]) ⇒ Array.<number> +helper for [retrieve](#FilterDefinition.retrieve) + +**Kind**: static method of [FilterDefinition](#FilterDefinition) +**Returns**: Array.<number> - Array of folder IDs + +| Param | Type | Default | Description | +| --- | --- | --- | --- | +| [hidden] | boolean | false | used to filter out hidden or non-hidden filterDefinitions | + + + +### FilterDefinition.getMeasureFolderIds() ⇒ Array.<number> +helper for [retrieve](#FilterDefinition.retrieve) + +**Kind**: static method of [FilterDefinition](#FilterDefinition) +**Returns**: Array.<number> - Array of folder IDs + + +### FilterDefinition.cacheDeFields(metadataTypeMapObj) +**Kind**: static method of [FilterDefinition](#FilterDefinition) + +| Param | Type | Description | +| --- | --- | --- | +| metadataTypeMapObj | TYPE.MultiMetadataTypeMap | - | + + + +### FilterDefinition.cacheContactAttributes(metadataTypeMapObj) +**Kind**: static method of [FilterDefinition](#FilterDefinition) + +| Param | Type | Description | +| --- | --- | --- | +| metadataTypeMapObj | TYPE.MultiMetadataTypeMap | - | + + + +### FilterDefinition.cacheMeasures(metadataTypeMapObj) +**Kind**: static method of [FilterDefinition](#FilterDefinition) + +| Param | Type | Description | +| --- | --- | --- | +| metadataTypeMapObj | TYPE.MultiMetadataTypeMap | - | + ### FilterDefinition.retrieveForCache() ⇒ Promise.<{metadata: TYPE.FilterDefinitionMap, type: string}> @@ -2992,6 +3035,27 @@ parses retrieved Metadata before saving | --- | --- | --- | | metadata | TYPE.FilterDefinitionItem | a single record | + + +### FilterDefinition.resolveFieldIds(metadata, [fieldCache], [filter]) ⇒ void +**Kind**: static method of [FilterDefinition](#FilterDefinition) + +| Param | Type | Description | +| --- | --- | --- | +| metadata | TYPE.FilterDefinitionItem | - | +| [fieldCache] | Array.<object> | - | +| [filter] | object | - | + + + +### FilterDefinition.resolveAttributeIds(metadata, [filter]) ⇒ void +**Kind**: static method of [FilterDefinition](#FilterDefinition) + +| Param | Type | Description | +| --- | --- | --- | +| metadata | TYPE.FilterDefinitionItem | - | +| [filter] | object | - | + ### FilterDefinition.preDeployTasks(metadata) ⇒ Promise.<TYPE.FilterDefinitionItem> @@ -3028,6 +3092,20 @@ Updates a single item | --- | --- | --- | | metadata | TYPE.FilterDefinitionItem | a single item | + + +## FilterDefinitionHidden ⇐ [FilterDefinitionHidden](#FilterDefinitionHidden) +FilterDefinitionHidden MetadataType + +**Kind**: global class +**Extends**: [FilterDefinitionHidden](#FilterDefinitionHidden) + + +### FilterDefinitionHidden.getFilterFolderIds() ⇒ Array.<number> +helper for [retrieve](#FilterDefinition.retrieve) + +**Kind**: static method of [FilterDefinitionHidden](#FilterDefinitionHidden) +**Returns**: Array.<number> - Array of folder IDs ## Folder ⇐ [MetadataType](#MetadataType) diff --git a/lib/MetadataTypeDefinitions.js b/lib/MetadataTypeDefinitions.js index cba29b1db..68da505fa 100644 --- a/lib/MetadataTypeDefinitions.js +++ b/lib/MetadataTypeDefinitions.js @@ -23,6 +23,7 @@ const MetadataTypeDefinitions = { fileTransfer: require('./metadataTypes/definitions/FileTransfer.definition'), filter: require('./metadataTypes/definitions/Filter.definition'), filterDefinition: require('./metadataTypes/definitions/FilterDefinition.definition'), + filterDefinitionHidden: require('./metadataTypes/definitions/FilterDefinitionHidden.definition'), folder: require('./metadataTypes/definitions/Folder.definition'), importFile: require('./metadataTypes/definitions/ImportFile.definition'), journey: require('./metadataTypes/definitions/Journey.definition'), diff --git a/lib/MetadataTypeInfo.js b/lib/MetadataTypeInfo.js index f88326649..6f4112bc4 100644 --- a/lib/MetadataTypeInfo.js +++ b/lib/MetadataTypeInfo.js @@ -23,6 +23,7 @@ const MetadataTypeInfo = { fileTransfer: require('./metadataTypes/FileTransfer'), filter: require('./metadataTypes/Filter'), filterDefinition: require('./metadataTypes/FilterDefinition'), + filterDefinitionHidden: require('./metadataTypes/FilterDefinitionHidden'), folder: require('./metadataTypes/Folder'), importFile: require('./metadataTypes/ImportFile'), journey: require('./metadataTypes/Journey'), diff --git a/lib/metadataTypes/DataExtension.js b/lib/metadataTypes/DataExtension.js index 6e96f463b..9676723ae 100644 --- a/lib/metadataTypes/DataExtension.js +++ b/lib/metadataTypes/DataExtension.js @@ -1341,7 +1341,7 @@ class DataExtension extends MetadataType { * @returns {Promise.<{metadata: TYPE.DataExtensionMap, type: string}>} Promise */ static async retrieveForCache() { - return this.retrieve(null, ['ObjectID', 'CustomerKey', 'Name'], this.buObject, null, null); + return this.retrieve(null, ['ObjectID', 'CustomerKey', 'Name']); } /** * Retrieves dataExtension metadata in template format. diff --git a/lib/metadataTypes/Filter.js b/lib/metadataTypes/Filter.js index 4a09dc1fa..214a6c3c6 100644 --- a/lib/metadataTypes/Filter.js +++ b/lib/metadataTypes/Filter.js @@ -33,32 +33,17 @@ class Filter extends MetadataType { static async retrieve(retrieveDir, _, __, key) { return super.retrieveREST(retrieveDir, '/automation/v1/filters/', null, key); } - /** - * manages post retrieve steps - * - * @param {TYPE.FilterItem} item a single record - * @returns {TYPE.FilterItem} parsed metadata definition - */ - static postRetrieveTasks(item) { - return this.parseMetadata(item); - } /** * parses retrieved Metadata before saving * * @param {TYPE.FilterItem} metadata a single record * @returns {TYPE.FilterItem} parsed metadata definition */ - static parseMetadata(metadata) { - try { - // folder - metadata.r__folder_Path = cache.searchForField( - 'folder', - metadata.categoryId, - 'ID', - 'Path' - ); - delete metadata.categoryId; + static postRetrieveTasks(metadata) { + // folder + this.setFolderPath(metadata); + try { // filterDefinition metadata.r__filterDefinition_CustomerKey = cache.searchForField( 'filterDefinition', @@ -67,7 +52,21 @@ class Filter extends MetadataType { 'key' ); delete metadata.filterDefinitionId; - + } catch { + try { + // filterDefinition + metadata.r__filterDefinition_CustomerKey = cache.searchForField( + 'filterDefinitionHidden', + metadata.filterDefinitionId, + 'id', + 'key' + ); + delete metadata.filterDefinitionId; + } catch { + // ignore + } + } + try { // source if (metadata.sourceTypeId === 1) { // list @@ -90,7 +89,12 @@ class Filter extends MetadataType { }` ); } - + } catch (ex) { + Util.logger.warn( + ` - filter '${metadata.name}' (${metadata.customerKey}): Destination not found (${ex.message})` + ); + } + try { // target if (metadata.destinationTypeId === 1) { // list @@ -106,7 +110,7 @@ class Filter extends MetadataType { delete metadata.destinationTypeId; } else { Util.logger.warn( - ` - Filter '${metadata.name}' (${ + ` - filter '${metadata.name}' (${ metadata.customerKey }): Unsupported destination type ${metadata.destinationTypeId}=${ dataTypes[metadata.destinationTypeId] @@ -115,7 +119,7 @@ class Filter extends MetadataType { } } catch (ex) { Util.logger.warn( - ` - Filter '${metadata.name}' (${metadata.customerKey}): ${ex.message}` + ` - filter '${metadata.name}' (${metadata.customerKey}): Source not found (${ex.message})` ); } return metadata; diff --git a/lib/metadataTypes/FilterDefinition.js b/lib/metadataTypes/FilterDefinition.js index 5422e72eb..6c480fed2 100644 --- a/lib/metadataTypes/FilterDefinition.js +++ b/lib/metadataTypes/FilterDefinition.js @@ -2,6 +2,8 @@ const TYPE = require('../../types/mcdev.d'); const MetadataType = require('./MetadataType'); +const DataExtensionField = require('./DataExtensionField'); +const Folder = require('./Folder'); const Util = require('../util/util'); const cache = require('../util/cache'); const { XMLBuilder, XMLParser } = require('fast-xml-parser'); @@ -12,6 +14,7 @@ const { XMLBuilder, XMLParser } = require('fast-xml-parser'); * @augments MetadataType */ class FilterDefinition extends MetadataType { + static cache = {}; // type internal cache for various things /** * Retrieves all records and saves it to disk * @@ -22,74 +25,223 @@ class FilterDefinition extends MetadataType { * @returns {Promise.<{metadata: TYPE.FilterDefinitionMap, type: string}>} Promise of items */ static async retrieve(retrieveDir, _, __, key) { - // #1 get the list via SOAP cause the corresponding REST call has no BU filter apparently - // for reference the rest path: '/automation/v1/filterdefinitions?view=categoryinfo' - - const soapFields = ['DataFilter', 'ObjectID', 'CustomerKey', 'Name']; - let requestParams; - if (key) { - requestParams = { - filter: { - leftOperand: 'CustomerKey', - operator: 'equals', - rightOperand: key, - }, - }; - } - - /** - * @type {TYPE.FilterDefinitionSOAPItemMap[]} - */ - const responseSOAP = await this.client.soap.retrieveBulk( - this.definition.type, - soapFields, - requestParams - ); + const filterFolders = await this.getFilterFolderIds(); - // backup REST value of the keyField - const keyFieldBak = this.definition.keyField; - this.definition.keyField = 'CustomerKey'; - const responseSOAPMap = this.parseResponseBody(responseSOAP, key); - // restore the keyField to its REST value - this.definition.keyField = keyFieldBak; - - /** - * @type {TYPE.FilterDefinitionSOAPItem[]} - */ - const responseSOAPList = responseSOAP.Results.filter((item) => { - if (item.ObjectState) { - Util.logger.debug(`Filtered filterDefinition ${item.Name}: ${item.ObjectState}`); - return false; - } else { - return true; - } - }); - - // #2 - // /automation/v1/filterdefinitions/ - const metadataMap = ( - await super.retrieveRESTcollection( - responseSOAPList.map((item) => ({ - id: item.ObjectID, - uri: '/email/v1/filters/filterdefinition/' + item.ObjectID, - })) - ) - ).metadata; - for (const item of Object.values(metadataMap)) { - // description is not returned when empty + const metadataTypeMapObj = { metadata: {}, type: this.definition.type }; + for (const folderId of filterFolders) { + const metadataMapFolder = await super.retrieveREST( + null, + 'email/v1/filters/filterdefinition/category/' + + folderId + + '?derivedFromType=1,2,3,4&', + null, + key + ); + if (Object.keys(metadataMapFolder.metadata).length) { + metadataTypeMapObj.metadata = { + ...metadataTypeMapObj.metadata, + ...metadataMapFolder.metadata, + }; + if (key) { + // if key was found we can stop checking other folders + break; + } + } + } + // console.log('metadataMap', metadataMap); + + for (const item of Object.values(metadataTypeMapObj.metadata)) { + // description is not returned when emptyg item.description ||= ''; - // add extra info from XML - item.c__soap_DataFilter = responseSOAPMap[item.key].DataFilter; } if (retrieveDir) { - const savedMetadata = await this.saveResults(metadataMap, retrieveDir); + // custom dataExtensionField caching + await this.cacheDeFields(metadataTypeMapObj); + await this.cacheContactAttributes(metadataTypeMapObj); + await this.cacheMeasures(metadataTypeMapObj); + + const savedMetadata = await this.saveResults(metadataTypeMapObj.metadata, retrieveDir); Util.logger.info( - `Downloaded: ${this.definition.type} (${Object.keys(savedMetadata).length})` + `Downloaded: ${this.definition.type} (${Object.keys(savedMetadata).length})` + + Util.getKeysString(key) ); } - return { metadata: metadataMap, type: this.definition.type }; + return metadataTypeMapObj; + } + /** + * helper for {@link FilterDefinition.retrieve} + * + * @param {boolean} [hidden] used to filter out hidden or non-hidden filterDefinitions + * @returns {number[]} Array of folder IDs + */ + static async getFilterFolderIds(hidden = false) { + const fromCache = + this.cache.folderFilter || cache.getCache().folder + ? Object.values(this.cache.folderFilter || cache.getCache().folder) + .filter((item) => item.ContentType === 'filterdefinition') + .filter( + (item) => + (!hidden && item.Path.startsWith('Data Filters')) || + (hidden && !item.Path.startsWith('Data Filters')) + ) // only retrieve from Data Filters folder + .map((item) => item.ID) + : []; + if (fromCache.length) { + return fromCache; + } + + const subTypeArr = ['hidden', 'filterdefinition']; + Util.logger.info(` - Caching dependent Metadata: folder`); + Util.logSubtypes(subTypeArr); + + Folder.client = this.client; + Folder.buObject = this.buObject; + Folder.properties = this.properties; + this.cache.folderFilter = (await Folder.retrieveForCache(null, subTypeArr)).metadata; + return this.getFilterFolderIds(hidden); + } + /** + * helper for {@link FilterDefinition.retrieve} + * + * @returns {number[]} Array of folder IDs + */ + static async getMeasureFolderIds() { + const fromCache = + this.cache.folderMeasure?.[this.buObject.mid] || cache.getCache().folder + ? Object.values( + this.cache.folderMeasure?.[this.buObject.mid] || cache.getCache().folder + ) + .filter((item) => item.ContentType === 'measure') + .map((item) => item.ID) + : []; + if (fromCache.length) { + return fromCache; + } + + const subTypeArr = ['measure']; + Util.logger.info(` - Caching dependent Metadata: folder`); + Util.logSubtypes(subTypeArr); + + Folder.client = this.client; + Folder.buObject = this.buObject; + Folder.properties = this.properties; + this.cache.folderMeasure ||= {}; + this.cache.folderMeasure[this.buObject.mid] = ( + await Folder.retrieveForCache(null, subTypeArr) + ).metadata; + return this.getMeasureFolderIds(); + } + + /** + * + * @param {TYPE.MultiMetadataTypeMap} metadataTypeMapObj - + */ + static async cacheDeFields(metadataTypeMapObj) { + const deKeys = Object.values(metadataTypeMapObj.metadata) + .filter((item) => item.derivedFromObjectTypeName === 'DataExtension') + .filter((item) => item.derivedFromObjectId) + .map((item) => { + try { + const deKey = cache.searchForField( + 'dataExtension', + item.derivedFromObjectId, + 'ObjectID', + 'CustomerKey' + ); + if (deKey) { + this.deIdKeyMap ||= {}; + this.deIdKeyMap[item.derivedFromObjectId] = deKey; + return deKey; + } + } catch { + return null; + } + }) + .filter(Boolean); + if (deKeys.length) { + Util.logger.info(' - Caching dependent Metadata: dataExtensionField'); + // only proceed with the download if we have dataExtension keys + const fieldOptions = {}; + for (const deKey of deKeys) { + fieldOptions.filter = fieldOptions.filter + ? { + leftOperand: { + leftOperand: 'DataExtension.CustomerKey', + operator: 'equals', + rightOperand: deKey, + }, + operator: 'OR', + rightOperand: fieldOptions.filter, + } + : { + leftOperand: 'DataExtension.CustomerKey', + operator: 'equals', + rightOperand: deKey, + }; + } + DataExtensionField.buObject = this.buObject; + DataExtensionField.client = this.client; + DataExtensionField.properties = this.properties; + this.dataExtensionFieldCache = ( + await DataExtensionField.retrieveForCache(fieldOptions, ['Name', 'ObjectID']) + ).metadata; + } } + /** + * + * @param {TYPE.MultiMetadataTypeMap} metadataTypeMapObj - + */ + static async cacheContactAttributes(metadataTypeMapObj) { + if (this.cache.contactAttributes?.[this.buObject.mid]) { + return; + } + const subscriberFilters = Object.values(metadataTypeMapObj.metadata) + .filter((item) => item.derivedFromObjectTypeName === 'SubscriberAttributes') + .filter((item) => item.derivedFromObjectId); + if (subscriberFilters.length) { + Util.logger.info(' - Caching dependent Metadata: contactAttributes'); + const response = await this.client.rest.get('/email/v1/Contacts/Attributes/'); + const keyFieldBackup = this.definition.keyField; + this.definition.keyField = 'id'; + this.cache.contactAttributes ||= {}; + this.cache.contactAttributes[this.buObject.mid] = this.parseResponseBody(response); + this.definition.keyField = keyFieldBackup; + } + } + /** + * + * @param {TYPE.MultiMetadataTypeMap} metadataTypeMapObj - + */ + static async cacheMeasures(metadataTypeMapObj) { + if (this.cache.measures?.[this.buObject.mid]) { + return; + } + const subscriberFilters = Object.values(metadataTypeMapObj.metadata) + .filter((item) => item.derivedFromObjectTypeName === 'SubscriberAttributes') + .filter((item) => item.derivedFromObjectId); + const measureFolders = await this.getMeasureFolderIds(); + if (subscriberFilters.length) { + Util.logger.info(' - Caching dependent Metadata: measure'); + const response = { items: [] }; + for (const folderId of measureFolders) { + const metadataMapFolder = await this.client.rest.getBulk( + 'email/v1/Measures/category/' + folderId + '/', + 250 // 250 is what the GUI is using + ); + if (Object.keys(metadataMapFolder.items).length) { + response.items.push(...metadataMapFolder.items); + } + } + + const keyFieldBackup = this.definition.keyField; + this.definition.keyField = 'measureID'; + this.cache.measures ||= {}; + this.cache.measures[this.buObject.mid] = this.parseResponseBody(response); + this.definition.keyField = keyFieldBackup; + } + } + /** * Retrieves all records for caching * @@ -111,57 +263,200 @@ class FilterDefinition extends MetadataType { // type 6 seems to be journey related. Maybe we need to change that again in the future return; } - try { - // folder - this.setFolderPath(metadata); + // folder + this.setFolderPath(metadata); - switch (metadata.derivedFromType) { - case 1: { - // SubscriberAttributes - // TODO - break; + // parse XML filter for further processing in JSON format + const xmlToJson = new XMLParser({ ignoreAttributes: false }); + metadata.c__filterDefinition = xmlToJson.parse( + metadata.filterDefinitionXml + )?.FilterDefinition; + delete metadata.filterDefinitionXml; + + switch (metadata.derivedFromType) { + case 1: { + if (metadata.c__filterDefinition['@_Source'] === 'SubscriberAttribute') { + if ( + metadata.derivedFromObjectId && + metadata.derivedFromObjectId !== '00000000-0000-0000-0000-000000000000' + ) { + // Lists + try { + metadata.r__source_list_PathName = cache.getListPathName( + metadata.derivedFromObjectId, + 'ObjectID' + ); + } catch { + Util.logger.warn( + ` - skipping ${this.definition.type} ${metadata.key}: list ${metadata.derivedFromObjectId} not found on current or Parent BU` + ); + // return; + } + } else { + // SubscriberAttributes + // - nothing to do + } } - case 2: { + + break; + } + case 2: { + // DataExtension + XXX? + if ( + metadata.c__filterDefinition['@_Source'] === 'Meta' || + metadata.derivedFromObjectId === '00000000-0000-0000-0000-000000000000' + ) { + // TODO - weird so far not understood case of Source=Meta + // sample: + } else if (metadata.c__filterDefinition['@_Source'] === 'DataExtension') { // DataExtension - metadata.r__dataExtension_CustomerKey = cache.searchForField( - 'dataExtension', - metadata.derivedFromObjectId, - 'ObjectID', - 'CustomerKey' - ); + try { + metadata.r__source_dataExtension_CustomerKey = + this.deIdKeyMap?.[metadata.derivedFromObjectId] || + cache.searchForField( + 'dataExtension', + metadata.derivedFromObjectId, + 'ObjectID', + 'CustomerKey' + ); + } catch { + Util.logger.debug( + ` - skipping ${this.definition.type} ${metadata.key}: dataExtension ${metadata.derivedFromObjectId} not found on BU` + ); + return; + } + } + break; + } + case 3: { + // TODO + break; + } + case 4: { + // TODO + break; + } + case 5: { + // TODO + break; + } + case 6: { + // TODO + break; + } + } + + // map Condition ID to fields ID + switch (metadata.derivedFromType) { + case 1: { + // SubscriberAttributes + this.resolveAttributeIds(metadata); + delete metadata.derivedFromObjectId; + delete metadata.derivedFromType; + delete metadata.c__filterDefinition['@_Source']; + break; + } + case 2: { + if (metadata.c__filterDefinition['@_Source'] === 'Meta') { + // TODO - weird so far not understood case of Source=Meta + // sample: + } else if (metadata.c__filterDefinition['@_Source'] === 'DataExtension') { + // DataExtension + this.resolveFieldIds(metadata); delete metadata.derivedFromObjectId; delete metadata.derivedFromType; - break; - } - case 3: { - // TODO - break; - } - case 4: { - // TODO - break; - } - case 5: { - // TODO - break; - } - case 6: { - // TODO - break; + delete metadata.c__filterDefinition['@_Source']; + delete metadata.c__filterDefinition['@_SourceID']; } + break; + } + case 3: { + // TODO + break; + } + case 4: { + // TODO + break; } + case 5: { + // TODO + break; + } + case 6: { + // TODO + break; + } + } + return metadata; + } - const xmlToJson = new XMLParser({ ignoreAttributes: false }); - metadata.c__filterDefinition = xmlToJson.parse(metadata.filterDefinitionXml); - // TODO map Condition ID to DataExtensionField ID - delete metadata.filterDefinitionXml; - } catch (ex) { - Util.logger.error( - `FilterDefinition '${metadata.name}' (${metadata.key}): ${ex.message}` + /** + * + * @param {TYPE.FilterDefinitionItem} metadata - + * @param {object[]} [fieldCache] - + * @param {object} [filter] - + * @returns {void} + */ + static resolveFieldIds(metadata, fieldCache, filter) { + if (!filter) { + return this.resolveFieldIds( + metadata, + Object.values(this.dataExtensionFieldCache), + metadata.c__filterDefinition?.ConditionSet ); } - return metadata; + const conditionsArr = Array.isArray(filter.Condition) + ? filter.Condition + : [filter.Condition]; + for (const condition of conditionsArr) { + condition.r__dataExtensionField = fieldCache.find( + (field) => field.ObjectID === condition['@_ID'] + )?.Name; + delete condition['@_ID']; + if (['IsEmpty', 'IsNotEmpty'].includes(condition['@_Operator'])) { + delete condition.Value; + } + } + if (filter.ConditionSet) { + this.resolveFieldIds(metadata, fieldCache, filter.ConditionSet); + } } + /** + * + * @param {TYPE.FilterDefinitionItem} metadata - + * @param {object} [filter] - + * @returns {void} + */ + static resolveAttributeIds(metadata, filter) { + if (!filter) { + return this.resolveAttributeIds(metadata, metadata.c__filterDefinition?.ConditionSet); + } + const contactAttributes = this.cache.contactAttributes[this.buObject.mid]; + const measures = this.cache.measures[this.buObject.mid]; + const conditionsArr = Array.isArray(filter.Condition) + ? filter.Condition + : [filter.Condition]; + for (const condition of conditionsArr) { + condition['@_ID'] += ''; + if (condition['@_SourceType'] === 'Measure' && measures[condition['@_ID']]) { + condition.r__measure = measures[condition['@_ID']]?.name; + delete condition['@_ID']; + } else if ( + condition['@_SourceType'] !== 'Measure' && + contactAttributes[condition['@_ID']] + ) { + condition.r__contactAttribute = contactAttributes[condition['@_ID']]?.name; + delete condition['@_ID']; + } + if (['IsEmpty', 'IsNotEmpty'].includes(condition['@_Operator'])) { + delete condition.Value; + } + } + if (filter.ConditionSet) { + this.resolveAttributeIds(metadata, filter.ConditionSet); + } + } + /** * prepares a item for deployment * diff --git a/lib/metadataTypes/FilterDefinitionHidden.js b/lib/metadataTypes/FilterDefinitionHidden.js new file mode 100644 index 000000000..634fb9a44 --- /dev/null +++ b/lib/metadataTypes/FilterDefinitionHidden.js @@ -0,0 +1,24 @@ +'use strict'; + +// const TYPE = require('../../types/mcdev.d'); +const FilterDefinition = require('./FilterDefinition'); + +/** + * FilterDefinitionHidden MetadataType + * + * @augments FilterDefinitionHidden + */ +class FilterDefinitionHidden extends FilterDefinition { + /** + * helper for {@link FilterDefinition.retrieve} + * + * @returns {number[]} Array of folder IDs + */ + static async getFilterFolderIds() { + return super.getFilterFolderIds(true); + } +} +// Assign definition to static attributes +FilterDefinitionHidden.definition = require('../MetadataTypeDefinitions').filterDefinitionHidden; + +module.exports = FilterDefinitionHidden; diff --git a/lib/metadataTypes/definitions/Filter.definition.js b/lib/metadataTypes/definitions/Filter.definition.js index 9f442bda1..51f50508b 100644 --- a/lib/metadataTypes/definitions/Filter.definition.js +++ b/lib/metadataTypes/definitions/Filter.definition.js @@ -1,8 +1,15 @@ module.exports = { bodyIteratorField: 'items', - dependencies: ['filterDefinition', 'list', 'dataExtension', 'folder'], + dependencies: [ + 'filterDefinition', + 'filterDefinitionHidden', + 'list', + 'dataExtension', + 'folder-filteractivity', + 'folder-hidden', + ], hasExtended: false, - idField: 'id', + idField: 'filterActivityId', keyIsFixed: null, keyField: 'customerKey', nameField: 'name', @@ -56,10 +63,10 @@ module.exports = { template: true, }, filterActivityId: { - isCreateable: null, - isUpdateable: null, - retrieving: true, - template: true, + isCreateable: false, + isUpdateable: true, + retrieving: false, + template: false, }, filterDefinitionId: { isCreateable: true, diff --git a/lib/metadataTypes/definitions/FilterDefinition.definition.js b/lib/metadataTypes/definitions/FilterDefinition.definition.js index 83f51d78a..b8299c046 100644 --- a/lib/metadataTypes/definitions/FilterDefinition.definition.js +++ b/lib/metadataTypes/definitions/FilterDefinition.definition.js @@ -1,5 +1,5 @@ module.exports = { - bodyIteratorField: 'Results', + bodyIteratorField: 'items', dependencies: ['folder-filterdefinition', 'folder-hidden', 'dataExtension'], filter: {}, hasExtended: false, @@ -12,10 +12,10 @@ module.exports = { createdNameField: 'createdBy', lastmodDateField: 'lastUpdated', lastmodNameField: 'lastUpdatedBy', - restPagination: false, + restPagination: true, + restPageSize: 100, type: 'filterDefinition', - typeDescription: - 'Defines an audience based on specified rules. Used by Filter Activities and Filtered DEs.', + typeDescription: 'Defines an audience based on specified rules. Used by Filter Activities.', typeRetrieveByDefault: true, typeName: 'Filter Definition', fields: { @@ -44,12 +44,13 @@ module.exports = { retrieving: false, template: false, }, - // createdByName: { - // isCreateable: false, - // isUpdateable: false, - // retrieving: false, - // template: false, - // }, + createdByName: { + // actual name of user indicated by id in createdBy + isCreateable: false, + isUpdateable: false, + retrieving: false, + template: false, + }, lastUpdated: { isCreateable: false, isUpdateable: false, @@ -62,12 +63,13 @@ module.exports = { retrieving: false, template: false, }, - // lastUpdatedName: { - // isCreateable: false, - // isUpdateable: false, - // retrieving: false, - // template: false, - // }, + lastUpdatedByName: { + // actual name of user indicated by id in lastUpdatedBy + isCreateable: false, + isUpdateable: false, + retrieving: false, + template: false, + }, name: { isCreateable: true, isUpdateable: true, @@ -81,13 +83,12 @@ module.exports = { retrieving: true, template: true, }, - // CategoryId: { - // // used by UPDATE payload - // isCreateable: false, - // isUpdateable: true, - // retrieving: false, - // template: false, - // }, + description: { + isCreateable: true, + isUpdateable: true, + retrieving: true, + template: true, + }, filterDefinitionXml: { isCreateable: true, isUpdateable: true, @@ -126,8 +127,8 @@ module.exports = { // dataExtension name; field only returned by GET-API isCreateable: false, isUpdateable: false, - retrieving: false, - template: false, + retrieving: true, + template: true, }, isSendable: { isCreateable: false, // automatically set during create @@ -135,5 +136,20 @@ module.exports = { retrieving: true, template: true, }, + r__dataExtension_CustomerKey: { + isCreateable: false, + isUpdateable: false, + retrieving: true, + template: true, + }, + c__filterDefinition: { + skipValidation: true, + }, + r__folder_Path: { + isCreateable: false, + isUpdateable: false, + retrieving: true, + template: true, + }, }, }; diff --git a/lib/metadataTypes/definitions/FilterDefinitionHidden.definition.js b/lib/metadataTypes/definitions/FilterDefinitionHidden.definition.js new file mode 100644 index 000000000..60b8a7c70 --- /dev/null +++ b/lib/metadataTypes/definitions/FilterDefinitionHidden.definition.js @@ -0,0 +1,156 @@ +module.exports = { + bodyIteratorField: 'items', + dependencies: ['folder-filterdefinition', 'folder-hidden', 'dataExtension', 'list'], + filter: {}, + hasExtended: false, + idField: 'id', + keyField: 'key', + nameField: 'name', + folderType: 'filterdefinition', + folderIdField: 'categoryId', + createdDateField: 'createdDate', + createdNameField: 'createdBy', + lastmodDateField: 'lastUpdated', + lastmodNameField: 'lastUpdatedBy', + restPagination: true, + restPageSize: 100, + type: 'filterDefinitionHidden', + typeDescription: + 'Defines an audience based on specified rules. Used by filtered DEs and filtered Lists.', + typeRetrieveByDefault: false, + typeName: 'Filter Definition for filtered Lists && Data Extensions', + fields: { + // the GUI seems to ONLY send fields during update that are actually changed. It has yet to be tested if that also works with sending other fiedls as well + id: { + isCreateable: false, + isUpdateable: false, // included in URL + retrieving: false, + template: false, + }, + key: { + isCreateable: true, + isUpdateable: true, + retrieving: true, + template: true, + }, + createdDate: { + isCreateable: false, + isUpdateable: false, + retrieving: false, + template: false, + }, + createdBy: { + isCreateable: false, + isUpdateable: false, + retrieving: false, + template: false, + }, + createdByName: { + // actual name of user indicated by id in createdBy + isCreateable: false, + isUpdateable: false, + retrieving: false, + template: false, + }, + lastUpdated: { + isCreateable: false, + isUpdateable: false, + retrieving: false, + template: false, + }, + lastUpdatedBy: { + isCreateable: false, + isUpdateable: false, + retrieving: false, + template: false, + }, + lastUpdatedByName: { + // actual name of user indicated by id in lastUpdatedBy + isCreateable: false, + isUpdateable: false, + retrieving: false, + template: false, + }, + name: { + isCreateable: true, + isUpdateable: true, + retrieving: true, + template: true, + }, + categoryId: { + // returned by GET / CREATE / UPDATE; used in CREATE payload + isCreateable: true, + isUpdateable: true, + retrieving: true, + template: true, + }, + description: { + isCreateable: true, + isUpdateable: true, + retrieving: true, + template: true, + }, + filterDefinitionXml: { + isCreateable: true, + isUpdateable: true, + retrieving: true, + template: true, + }, + // DerivedFromType: { + // // this upper-cased spelling is used by GUI when creating a dataExtension based filterDefintion + // isCreateable: true, + // isUpdateable: false, // cannot be updated + // retrieving: false, + // template: false, + // }, + derivedFromType: { + // 1: SubscriberAttributes, 2: DataExtension, 6: EntryCriteria; + isCreateable: true, + isUpdateable: false, // cannot be updated + retrieving: true, + template: true, + }, + derivedFromObjectId: { + // dataExtension ID or '00000000-0000-0000-0000-000000000000' for lists + isCreateable: true, + isUpdateable: false, // cannot be updated + retrieving: true, + template: true, + }, + derivedFromObjectTypeName: { + // "SubscriberAttributes" | "DataExtension" | "EntryCriteria" ...; only returned by GET API + isCreateable: false, + isUpdateable: false, + retrieving: true, + template: true, + }, + derivedFromObjectName: { + // dataExtension name; field only returned by GET-API + isCreateable: false, + isUpdateable: false, + retrieving: true, + template: true, + }, + isSendable: { + isCreateable: false, // automatically set during create + isUpdateable: false, + retrieving: true, + template: true, + }, + r__dataExtension_CustomerKey: { + isCreateable: false, + isUpdateable: false, + retrieving: true, + template: true, + }, + c__filterDefinition: { + skipValidation: true, + }, + r__folder_Path: { + isCreateable: false, + isUpdateable: false, + retrieving: true, + template: true, + }, + }, +}; From 8cb78b059b648a84c8fec74dcb2f02db21754b6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Berkefeld?= Date: Wed, 30 Aug 2023 17:13:21 +0200 Subject: [PATCH 6/8] #9: make filters available to type automation --- lib/metadataTypes/definitions/Automation.definition.js | 1 + lib/metadataTypes/definitions/Filter.definition.js | 2 ++ 2 files changed, 3 insertions(+) diff --git a/lib/metadataTypes/definitions/Automation.definition.js b/lib/metadataTypes/definitions/Automation.definition.js index 656bcf659..d6054bb9f 100644 --- a/lib/metadataTypes/definitions/Automation.definition.js +++ b/lib/metadataTypes/definitions/Automation.definition.js @@ -27,6 +27,7 @@ module.exports = { 'dataExtract', 'emailSend', 'fileTransfer', + 'filter', 'folder-automations', 'importFile', 'query', diff --git a/lib/metadataTypes/definitions/Filter.definition.js b/lib/metadataTypes/definitions/Filter.definition.js index 51f50508b..5fbea7be4 100644 --- a/lib/metadataTypes/definitions/Filter.definition.js +++ b/lib/metadataTypes/definitions/Filter.definition.js @@ -13,6 +13,8 @@ module.exports = { keyIsFixed: null, keyField: 'customerKey', nameField: 'name', + folderType: 'filteractivity', + folderIdField: 'categoryId', createdDateField: 'createdDate', createdNameField: null, lastmodDateField: 'modifiedDate', From 45f8657ba77f75fa8f5f9f7d74b7dad9dc9d02cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Berkefeld?= Date: Wed, 30 Aug 2023 17:17:18 +0200 Subject: [PATCH 7/8] #325: save notificationEmailAddress as array of emails to enhance UX --- lib/metadataTypes/Verification.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/metadataTypes/Verification.js b/lib/metadataTypes/Verification.js index 93ce29fd0..6e967521b 100644 --- a/lib/metadataTypes/Verification.js +++ b/lib/metadataTypes/Verification.js @@ -189,6 +189,10 @@ class Verification extends MetadataType { 'ObjectID' ); delete metadata.r__dataExtension_CustomerKey; + metadata.notificationEmailAddress = Array.isArray(metadata.notificationEmailAddress) + ? metadata.notificationEmailAddress.map((item) => item.trim()).join(',') + : Array.isArray(metadata.notificationEmailAddress); + return metadata; } /** @@ -211,6 +215,9 @@ class Verification extends MetadataType { ` - ${this.definition.type} ${metadata[this.definition.keyField]}: ${ex.message}` ); } + metadata.notificationEmailAddress = metadata.notificationEmailAddress + .split(',') + .map((item) => item.trim()); return metadata; } /** From e0a4655cf2d91e42812c6bcd792b10e397eb6c8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Berkefeld?= Date: Wed, 30 Aug 2023 17:23:41 +0200 Subject: [PATCH 8/8] #325: adapt tests to change in 45f8657ba77f75fa8f5f9f7d74b7dad9dc9d02cb --- test/resources/9999999/verification/build-expected.json | 2 +- test/resources/9999999/verification/get-expected.json | 2 +- test/resources/9999999/verification/patch-expected.json | 2 +- test/resources/9999999/verification/post-expected.json | 2 +- test/resources/9999999/verification/template-expected.json | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/resources/9999999/verification/build-expected.json b/test/resources/9999999/verification/build-expected.json index f717cc626..98d32e01c 100644 --- a/test/resources/9999999/verification/build-expected.json +++ b/test/resources/9999999/verification/build-expected.json @@ -1,6 +1,6 @@ { "dataVerificationDefinitionId": "testTemplated_39f6a488-20eb-4ba0-b0b9", - "notificationEmailAddress": "", + "notificationEmailAddress": [""], "notificationEmailMessage": "", "r__dataExtension_CustomerKey": "testTemplated_dataExtension", "shouldEmailOnFailure": false, diff --git a/test/resources/9999999/verification/get-expected.json b/test/resources/9999999/verification/get-expected.json index 1f9c1825b..1be889a98 100644 --- a/test/resources/9999999/verification/get-expected.json +++ b/test/resources/9999999/verification/get-expected.json @@ -1,6 +1,6 @@ { "dataVerificationDefinitionId": "testExisting_39f6a488-20eb-4ba0-b0b9", - "notificationEmailAddress": "", + "notificationEmailAddress": [""], "notificationEmailMessage": "", "r__dataExtension_CustomerKey": "testExisting_dataExtension", "shouldEmailOnFailure": false, diff --git a/test/resources/9999999/verification/patch-expected.json b/test/resources/9999999/verification/patch-expected.json index dc8d975d2..dde339242 100644 --- a/test/resources/9999999/verification/patch-expected.json +++ b/test/resources/9999999/verification/patch-expected.json @@ -1,6 +1,6 @@ { "dataVerificationDefinitionId": "testExisting_39f6a488-20eb-4ba0-b0b9", - "notificationEmailAddress": "test@accenture.com", + "notificationEmailAddress": ["test@accenture.com"], "notificationEmailMessage": "", "r__dataExtension_CustomerKey": "testExisting_dataExtension", "shouldEmailOnFailure": true, diff --git a/test/resources/9999999/verification/post-expected.json b/test/resources/9999999/verification/post-expected.json index b06a77b76..41795da1f 100644 --- a/test/resources/9999999/verification/post-expected.json +++ b/test/resources/9999999/verification/post-expected.json @@ -6,6 +6,6 @@ "value2": 0, "shouldStopOnFailure": false, "shouldEmailOnFailure": false, - "notificationEmailAddress": "", + "notificationEmailAddress": [""], "notificationEmailMessage": "" } diff --git a/test/resources/9999999/verification/template-expected.json b/test/resources/9999999/verification/template-expected.json index 04296b7ea..f75056e69 100644 --- a/test/resources/9999999/verification/template-expected.json +++ b/test/resources/9999999/verification/template-expected.json @@ -1,6 +1,6 @@ { "dataVerificationDefinitionId": "{{{prefix}}}39f6a488-20eb-4ba0-b0b9", - "notificationEmailAddress": "", + "notificationEmailAddress": [""], "notificationEmailMessage": "", "r__dataExtension_CustomerKey": "{{{prefix}}}dataExtension", "shouldEmailOnFailure": false,