From 75d33fe9647526dedcb00e6677632364d5e48be2 Mon Sep 17 00:00:00 2001 From: yevhenii-moroziuk Date: Tue, 20 Aug 2024 13:46:57 +0300 Subject: [PATCH 1/8] HCK-7642: Create script generation --- forward_engineering/configs/templates.js | 30 +++++ forward_engineering/ddlProvider.js | 89 +++++++++----- forward_engineering/helpers/tableHelper.js | 58 +++++++++ .../entity_level/entityLevelConfig.json | 111 ++++++++++++++++++ 4 files changed, 259 insertions(+), 29 deletions(-) diff --git a/forward_engineering/configs/templates.js b/forward_engineering/configs/templates.js index 52e0c9f5..11a0aaec 100644 --- a/forward_engineering/configs/templates.js +++ b/forward_engineering/configs/templates.js @@ -9,6 +9,36 @@ module.exports = { '\t\t${column_definitions}' + '${out_of_line_constraints}\n' + '\t)${tableOptions};\n', + createDynamicTable: + 'CREATE OR REPLACE${transient} DYNAMIC TABLE\n' + + '\t${name}\n' + + '${column_definitions}' + + '${targetLag}' + + '${refreshMode}' + + '${initialize}' + + '${clusterKeys}' + + '${dataRetentionTime}' + + '${maxDataExtensionTime}' + + '${comment}' + + '${tagsStatement}' + + '${selectStatement};\n', + createDynamicIcebergTable: + 'CREATE DYNAMIC ICEBERG${transient} TABLE\n' + + '\t${name}\n' + + '${column_definitions}' + + '${targetLag}' + + '${warehouse}' + + '${externalVolume}' + + '${catalog}' + + '${baseLocation}' + + '${refreshMode}' + + '${initialize}' + + '${clusterKeys}' + + '${dataRetentionTime}' + + '${comment}' + + '${copyGrants}' + + '${tagsStatement}' + + '${selectStatement};\n', createLikeTable: 'CREATE TABLE IF NOT EXISTS ${name} LIKE ${source_table}${tableOptions};\n', createCloneTable: 'CREATE TABLE IF NOT EXISTS ${name} CLONE ${source_table}${tableOptions};\n', createAsSelect: 'CREATE TABLE IF NOT EXISTS ${name} AS ${selectStatement}${tableOptions};\n', diff --git a/forward_engineering/ddlProvider.js b/forward_engineering/ddlProvider.js index 79de9f8a..88b9e470 100644 --- a/forward_engineering/ddlProvider.js +++ b/forward_engineering/ddlProvider.js @@ -28,7 +28,7 @@ module.exports = (baseProvider, options, app) => { const scriptFormat = options?.targetScriptOptions?.keyword || FORMATS.SNOWSIGHT; const keyHelper = require('./helpers/keyHelper')(app); - const { getFileFormat, getCopyOptions, addOptions, getAtOrBefore, mergeKeys } = + const { getFileFormat, getCopyOptions, addOptions, getAtOrBefore, mergeKeys, getDynamicTableProps } = require('./helpers/tableHelper')(app); const getFormatTypeOptions = require('./helpers/getFormatTypeOptions')(app); const { getStageCopyOptions } = require('./helpers/getStageCopyOptions')(app); @@ -78,18 +78,15 @@ module.exports = (baseProvider, options, app) => { }); const getOutOfLineConstraints = ( - foreignKeyConstraints, - primaryKeyConstraints, - uniqueKeyConstraints, + foreignKeyConstraints = [], + primaryKeyConstraints = [], + uniqueKeyConstraints = [], isParentActivated, ) => { - const constraints = [] - .concat(foreignKeyConstraints || []) - .concat(primaryKeyConstraints) - .concat(uniqueKeyConstraints) - .map(constraint => + const constraints = [...foreignKeyConstraints, ...primaryKeyConstraints, ...uniqueKeyConstraints].map( + constraint => isParentActivated ? commentIfDeactivated(constraint.statement, constraint) : constraint.statement, - ); + ); return !_.isEmpty(constraints) ? ',\n\t\t' + constraints.join(',\n\t\t') : ''; }; @@ -328,7 +325,27 @@ module.exports = (baseProvider, options, app) => { isCaseSensitive: tableData.isCaseSensitive, }); - if (tableData.selectStatement) { + if (!!tableData.dynamic) { + const dynamicTableOptions = getDynamicTableProps({ + tableData, + tagsStatement, + clusterKeys, + comment, + dataRetentionTime, + copyGrants, + columnDefinitions, + }); + + const template = tableData.dynamicTableProps.iceberg + ? templates.createDynamicIcebergTable + : templates.createDynamicTable; + + return assignTemplates(template, { + name: tableData.fullName, + transient, + ...dynamicTableOptions, + }); + } else if (tableData.selectStatement) { return assignTemplates(templates.createAsSelect, { name: tableData.fullName, selectStatement: tableData.selectStatement, @@ -376,24 +393,6 @@ module.exports = (baseProvider, options, app) => { comment, ), - column_definitions: columnDefinitions, - out_of_line_constraints: getOutOfLineConstraints( - tableData.foreignKeyConstraints, - tableData.compositePrimaryKeys, - tableData.compositeUniqueKeys, - isActivated, - ), - }); - } else { - return assignTemplates(templates.createTable, { - name: tableData.fullName, - temporary: temporary, - transient: transient, - tableOptions: addOptions( - [clusterKeys, stageFileFormat, copyOptions, dataRetentionTime, copyGrants, tagsStatement], - comment, - ), - column_definitions: columnDefinitions, out_of_line_constraints: getOutOfLineConstraints( tableData.foreignKeyConstraints, @@ -403,6 +402,23 @@ module.exports = (baseProvider, options, app) => { ), }); } + + return assignTemplates(templates.createTable, { + name: tableData.fullName, + temporary, + transient, + tableOptions: addOptions( + [clusterKeys, stageFileFormat, copyOptions, dataRetentionTime, copyGrants, tagsStatement], + comment, + ), + column_definitions: columnDefinitions, + out_of_line_constraints: getOutOfLineConstraints( + tableData.foreignKeyConstraints, + tableData.compositePrimaryKeys, + tableData.compositeUniqueKeys, + isActivated, + ), + }); }, convertColumnDefinition(columnDefinition) { @@ -803,6 +819,21 @@ module.exports = (baseProvider, options, app) => { temporary: firstTab.temporary, transient: firstTab.transient, external: firstTab.external, + dynamic: firstTab.dynamic, + dynamicTableProps: { + iceberg: firstTab.iceberg, + warehouse: firstTab.warehouse, + targetLag: firstTab.targetLag, + refreshMode: firstTab.refreshMode, + initialize: firstTab.initialize, + query: firstTab.query, + externalVolume: firstTab.externalVolume, + catalog: firstTab.catalog, + baseLocation: firstTab.baseLocation, + maxDataExtensionTime: !isNaN(firstTab.MAX_DATA_EXTENSION_TIME_IN_DAYS) + ? firstTab.MAX_DATA_EXTENSION_TIME_IN_DAYS + : '', + }, selectStatement: firstTab.selectStatement, isCaseSensitive: firstTab.isCaseSensitive, clusteringKey: Array.isArray(firstTab.clusteringKey) diff --git a/forward_engineering/helpers/tableHelper.js b/forward_engineering/helpers/tableHelper.js index 58b925eb..b02cda77 100644 --- a/forward_engineering/helpers/tableHelper.js +++ b/forward_engineering/helpers/tableHelper.js @@ -47,11 +47,69 @@ module.exports = app => { const mergeKeys = keys => keys.map(key => `"${key.name}"`).join(', '); + function getTargetLag({ targetLagType, targetLagAmount, targetLagDownstream }) { + return `TARGET_LAG = ${targetLagDownstream ? 'DOWNSTREAM' : `'${targetLagAmount} ${targetLagType}'`}\n`; + } + + const getDynamicTableProps = ({ + tableData, + transient, + tagsStatement, + clusterKeys, + comment, + dataRetentionTime, + copyGrants, + columnDefinitions, + }) => { + if (!tableData.dynamicTableProps) { + return {}; + } + + const { selectStatement } = tableData; + const { + iceberg, + targetLag, + warehouse, + refreshMode, + initialize, + maxDataExtensionTime, + externalVolume, + catalog, + baseLocation, + } = tableData.dynamicTableProps; + + return { + targetLag: targetLag?.[0] ? getTargetLag(targetLag[0]) : '', + warehouse: warehouse ? `WAREHOUSE = ${warehouse}\n` : '', + selectStatement: selectStatement + ? `AS\n${selectStatement + .split('\n') + .map((it, i, arr) => (i === arr.length - 1 ? `\t${it}` : `\t${it}\n`)) + .join('')}` + : '', + externalVolume: externalVolume ? `EXTERNAL_VOLUME = ${externalVolume}\n` : '', + catalog: catalog ? `CATALOG = ${catalog}\n` : '', + baseLocation: baseLocation ? `BASE_LOCATION = ${baseLocation}\n` : '', + column_definitions: columnDefinitions ? `\t(\n\t\t${columnDefinitions}\n\t)\n` : '', + refreshMode: refreshMode ? `REFRESH_MODE = ${refreshMode}\n` : '', + initialize: initialize ? `INITIALIZE = ${initialize}\n` : '', + clusterKeys, + dataRetentionTime: dataRetentionTime ? `${dataRetentionTime.trim()}\n` : '', + maxDataExtensionTime: maxDataExtensionTime + ? `MAX_DATA_EXTENSION_TIME_IN_DAYS = ${maxDataExtensionTime}\n` + : '', + copyGrants: copyGrants ? `${copyGrants.trim()}\n` : '', + comment: comment ? `${comment.trim()}\n` : '', + tagsStatement: tagsStatement ? `${tagsStatement.trim()}\n` : '', + }; + }; + return { getFileFormat, getCopyOptions, addOptions, getAtOrBefore, mergeKeys, + getDynamicTableProps, }; }; diff --git a/properties_pane/entity_level/entityLevelConfig.json b/properties_pane/entity_level/entityLevelConfig.json index dbf9d540..2985abd7 100644 --- a/properties_pane/entity_level/entityLevelConfig.json +++ b/properties_pane/entity_level/entityLevelConfig.json @@ -154,6 +154,83 @@ making sure that you maintain a proper JSON format. "value": "Vault" } }, + { + "propertyName": "As Dynamic", + "propertyKeyword": "dynamic", + "propertyType": "checkbox" + }, + { + "propertyName": "Iceberg", + "propertyKeyword": "iceberg", + "propertyType": "checkbox", + "dependency": { + "key": "dynamic", + "value": true + } + }, + { + "propertyName": "External volume", + "propertyKeyword": "externalVolume", + "propertyType": "text", + "dependency": { + "key": "iceberg", + "value": true + } + }, + { + "propertyName": "Catalog", + "propertyKeyword": "catalog", + "propertyType": "text", + "dependency": { + "key": "iceberg", + "value": true + } + }, + { + "propertyName": "Base location", + "propertyKeyword": "baseLocation", + "propertyType": "text", + "dependency": { + "key": "iceberg", + "value": true + } + }, + { + "propertyName": "Target Lag", + "propertyKeyword": "targetLag", + "propertyType": "group", + "structure": [ + { + "propertyName": "Type", + "propertyKeyword": "targetLagType", + "propertyType": "select", + "options": ["seconds", "minutes", "hours", "days"] + }, + { + "propertyName": "Amount", + "propertyKeyword": "targetLagAmount", + "propertyType": "numeric" + }, + { + "propertyName": "Downstream", + "propertyKeyword": "targetLagDownstream", + "propertyType": "checkbox" + } + ], + "dependency": { + "key": "dynamic", + "value": true + } + }, + { + "propertyName": "Warehouse", + "propertyKeyword": "warehouse", + "propertyType": "text", + "dependency": { + "key": "dynamic", + "value": true + } + }, { "propertyName": "Temporary", "propertyKeyword": "temporary", @@ -164,6 +241,26 @@ making sure that you maintain a proper JSON format. "propertyKeyword": "transient", "propertyType": "checkbox" }, + { + "propertyName": "Refresh Mode", + "propertyKeyword": "refreshMode", + "propertyType": "select", + "options": ["", "auto", "full", "incremental"], + "dependency": { + "key": "dynamic", + "value": true + } + }, + { + "propertyName": "Initialize", + "propertyKeyword": "initialize", + "propertyType": "select", + "options": ["", "on_create", "on_schedule"], + "dependency": { + "key": "dynamic", + "value": true + } + }, { "propertyName": "External", "propertyKeyword": "external", @@ -784,6 +881,20 @@ making sure that you maintain a proper JSON format. } } }, + { + "propertyName": "Max data extension in days", + "propertyKeyword": "MAX_DATA_EXTENSION_TIME_IN_DAYS", + "propertyTooltip": "Number of days for which Snowflake retains historical data for performing Time Travel actions (SELECT, CLONE, UNDROP) on the object. A value of 0 effectively disables Time Travel for the specified database, schema, or table", + "propertyValidate": true, + "propertyType": "numeric", + "valueType": "number", + "minValue": 0, + "maxValue": 90, + "dependency": { + "key": "dynamic", + "value": true + } + }, { "propertyName": "Like", "propertyKeyword": "like", From 88c33750cb4b38c831095564bd3515e5ddfeebc7 Mon Sep 17 00:00:00 2001 From: yevhenii-moroziuk Date: Tue, 20 Aug 2024 16:31:36 +0300 Subject: [PATCH 2/8] HCK-7642: add alter support --- forward_engineering/configs/templates.js | 4 ++-- forward_engineering/ddlProvider.js | 6 +++++- forward_engineering/helpers/alterScriptHelpers/common.js | 8 +++++++- .../helpers/alterScriptHelpers/commonScript.js | 4 ++-- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/forward_engineering/configs/templates.js b/forward_engineering/configs/templates.js index 11a0aaec..25da610a 100644 --- a/forward_engineering/configs/templates.js +++ b/forward_engineering/configs/templates.js @@ -67,11 +67,11 @@ module.exports = { 'CREATE${temporary} STAGE IF NOT EXISTS ${name} ${url}${storageIntegration}${credentials}${encryption};\n', alterSchema: 'ALTER SCHEMA IF EXISTS ${name} ${operation} ${options};', - alterTable: 'ALTER TABLE IF EXISTS ${name} ${action};', + alterTable: 'ALTER${dynamic} TABLE IF EXISTS ${name} ${action};', alterView: 'ALTER${materialized} VIEW IF EXISTS ${name} ${action};', alterSchemaScript: 'ALTER SCHEMA IF EXISTS ${name} ', - alterTableScript: 'ALTER TABLE IF EXISTS ${name} ', + alterTableScript: 'ALTER${dynamic} TABLE IF EXISTS ${name} ', alterEntityRename: 'RENAME TO ${name};\n', setPropertySchema: 'SET ${property};\n', unsetPropertySchema: 'UNSET ${property};\n', diff --git a/forward_engineering/ddlProvider.js b/forward_engineering/ddlProvider.js index 88b9e470..c1c90373 100644 --- a/forward_engineering/ddlProvider.js +++ b/forward_engineering/ddlProvider.js @@ -939,7 +939,10 @@ module.exports = (baseProvider, options, app) => { }, alterTable(data) { - const alterTableScript = getAlterEntityScript(templates.alterTableScript, data.nameData); + const alterTableScript = getAlterEntityScript(templates.alterTableScript, { + dynamic: data.dynamic, + ...data.nameData, + }); const { script } = _.flow( getAlterEntityRename(templates.alterTableScript, templates.alterEntityRename), getSetCollectionProperty(alterTableScript), @@ -969,6 +972,7 @@ module.exports = (baseProvider, options, app) => { return { ...data, + dynamic: collection.role.dynamic, formatTypeOptions: { ...data.formatTypeOptions, typeOptions: formatTypeOptions, diff --git a/forward_engineering/helpers/alterScriptHelpers/common.js b/forward_engineering/helpers/alterScriptHelpers/common.js index 5de147bb..4a13f415 100644 --- a/forward_engineering/helpers/alterScriptHelpers/common.js +++ b/forward_engineering/helpers/alterScriptHelpers/common.js @@ -1,7 +1,13 @@ const _ = require('lodash'); const { getDiffCopyOptionsByDefault } = require('./tableCopyOptionsHelper'); -const POSSIBLE_CHANGE_CONTAINER_DATA = ['DATA_RETENTION_TIME_IN_DAYS', 'description']; +const POSSIBLE_CHANGE_CONTAINER_DATA = [ + 'TARGET_LAG', + 'WAREHOUSE', + 'DATA_RETENTION_TIME_IN_DAYS', + 'MAX_DATA_EXTENSION_TIME_IN_DAYS', + 'description', +]; const REDUNDANT_OPTIONS = ['id']; const checkFieldPropertiesChanged = (compMod, propertiesToCheck) => { diff --git a/forward_engineering/helpers/alterScriptHelpers/commonScript.js b/forward_engineering/helpers/alterScriptHelpers/commonScript.js index 5b9672e4..d1626f9e 100644 --- a/forward_engineering/helpers/alterScriptHelpers/commonScript.js +++ b/forward_engineering/helpers/alterScriptHelpers/commonScript.js @@ -15,12 +15,12 @@ module.exports = ({ getName, getFullName, templates, assignTemplates, tab }) => return assignTemplates(templates.alterSchemaScript, { name: schemaFullName }); }; - const getAlterEntityScript = (template, { database, isCaseSensitive, newName, schemaName } = {}) => { + const getAlterEntityScript = (template, { dynamic, database, isCaseSensitive, newName, schemaName } = {}) => { const schemaFullName = getSchemaFullName(database, schemaName, isCaseSensitive); const tableName = getName(isCaseSensitive, newName); const tableFullName = getFullName(schemaFullName, tableName); - return assignTemplates(template, { name: tableFullName }); + return assignTemplates(template, { dynamic: dynamic ? ' DYNAMIC' : '', name: tableFullName }); }; const getAlterSchemaName = ({ script, data }) => { From c390e37be249780472728d1390154f311a2ea488 Mon Sep 17 00:00:00 2001 From: yevhenii-moroziuk Date: Tue, 20 Aug 2024 21:26:14 +0300 Subject: [PATCH 3/8] HCK-7642: update alter script generation --- .../helpers/alterScriptHelpers/common.js | 12 ++++--- .../alterScriptHelpers/commonScript.js | 31 +++++++++++++++---- 2 files changed, 32 insertions(+), 11 deletions(-) diff --git a/forward_engineering/helpers/alterScriptHelpers/common.js b/forward_engineering/helpers/alterScriptHelpers/common.js index 4a13f415..5687749c 100644 --- a/forward_engineering/helpers/alterScriptHelpers/common.js +++ b/forward_engineering/helpers/alterScriptHelpers/common.js @@ -1,13 +1,15 @@ const _ = require('lodash'); const { getDiffCopyOptionsByDefault } = require('./tableCopyOptionsHelper'); -const POSSIBLE_CHANGE_CONTAINER_DATA = [ - 'TARGET_LAG', - 'WAREHOUSE', +const POSSIBLE_UNSET_PROPERTIES = ['DATA_RETENTION_TIME_IN_DAYS', 'MAX_DATA_EXTENSION_TIME_IN_DAYS']; +const POSSIBLE_SET_PROPERTIES = [ + 'targetLag', + 'warehouse', 'DATA_RETENTION_TIME_IN_DAYS', 'MAX_DATA_EXTENSION_TIME_IN_DAYS', 'description', ]; +const POSSIBLE_CHANGE_CONTAINER_DATA = _.uniq([...POSSIBLE_UNSET_PROPERTIES, ...POSSIBLE_SET_PROPERTIES]); const REDUNDANT_OPTIONS = ['id']; const checkFieldPropertiesChanged = (compMod, propertiesToCheck) => { @@ -113,7 +115,7 @@ const prepareAlterSetUnsetData = ({ collection, data }) => { return acc; } - if (newData) { + if (newData && POSSIBLE_SET_PROPERTIES.includes(property)) { return { ...acc, set: { @@ -121,7 +123,7 @@ const prepareAlterSetUnsetData = ({ collection, data }) => { [property]: newData, }, }; - } else if (oldData) { + } else if (oldData && POSSIBLE_UNSET_PROPERTIES.includes(property)) { return { ...acc, unset: [...acc.unset, property], diff --git a/forward_engineering/helpers/alterScriptHelpers/commonScript.js b/forward_engineering/helpers/alterScriptHelpers/commonScript.js index d1626f9e..f13df298 100644 --- a/forward_engineering/helpers/alterScriptHelpers/commonScript.js +++ b/forward_engineering/helpers/alterScriptHelpers/commonScript.js @@ -41,6 +41,28 @@ module.exports = ({ getName, getFullName, templates, assignTemplates, tab }) => return { script, data }; }; + const mapKey = { + description: 'COMMENT', + targetLag: 'TARGET_LAG', + warehouse: 'WAREHOUSE', + DATA_RETENTION_TIME_IN_DAYS: 'DATA_RETENTION_TIME_IN_DAYS', + MAX_DATA_EXTENSION_TIME_IN_DAYS: 'MAX_DATA_EXTENSION_TIME_IN_DAYS', + }; + + const getValue = ({ key, propValue, data }, operation) => { + if (key === 'description') { + const scriptFormat = _.get(data, 'options.targetScriptOptions.keyword'); + + return escapeString(scriptFormat, propValue); + } else if (key === 'targetLag') { + const { targetLagAmount, targetLagType, targetLagDownstream } = data[operation].targetLag?.[0]; + + return `${targetLagDownstream ? 'DOWNSTREAM' : `'${targetLagAmount} ${targetLagType}'`}\n`; + } + + return propValue; + }; + const getSetCollectionProperty = alterScript => ({ script, data }) => { @@ -51,11 +73,8 @@ module.exports = ({ getName, getFullName, templates, assignTemplates, tab }) => const setPropertyData = Object.keys(setProperty).map((key, index) => { const propValue = setProperty[key]; - const scriptFormat = _.get(data, 'options.targetScriptOptions.keyword'); - - const value = key === 'description' ? escapeString(scriptFormat, propValue) : propValue; - key = key === 'description' ? 'COMMENT' : key; - const statement = `${key} = ${value}`; + const value = getValue({ key, data, propValue }, 'setProperty'); + const statement = `${mapKey[key]} = ${value}`; return Boolean(index) ? tab(statement) : statement; }); @@ -79,7 +98,7 @@ module.exports = ({ getName, getFullName, templates, assignTemplates, tab }) => } const unsetPropertyData = unsetProperty.map((key, index) => { - key = key === 'description' ? 'COMMENT' : key; + key = mapKey[key]; return Boolean(index) ? `${key}` : `${key}`; }); From faa4003e1da2aadc8673a0187fba576d9e8e96f1 Mon Sep 17 00:00:00 2001 From: yevhenii-moroziuk Date: Wed, 21 Aug 2024 12:05:57 +0300 Subject: [PATCH 4/8] HCK-7642: small improvements --- forward_engineering/ddlProvider.js | 5 +++++ forward_engineering/helpers/alterScriptHelpers/common.js | 2 +- .../helpers/alterScriptHelpers/commonScript.js | 8 ++++---- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/forward_engineering/ddlProvider.js b/forward_engineering/ddlProvider.js index c1c90373..eb69cc3a 100644 --- a/forward_engineering/ddlProvider.js +++ b/forward_engineering/ddlProvider.js @@ -939,6 +939,10 @@ module.exports = (baseProvider, options, app) => { }, alterTable(data) { + if (data.iceberg) { + return '// Dynamic Iceberg tables are currently only supported for CREATE statements. Specifying DYNAMIC ICEBERG in any other command (for example, ALTER DYNAMIC ICEBERG TABLE ) results in an error.'; + } + const alterTableScript = getAlterEntityScript(templates.alterTableScript, { dynamic: data.dynamic, ...data.nameData, @@ -973,6 +977,7 @@ module.exports = (baseProvider, options, app) => { return { ...data, dynamic: collection.role.dynamic, + iceberg: collection.compMod?.iceberg?.old || collection.compMod?.iceberg?.new, formatTypeOptions: { ...data.formatTypeOptions, typeOptions: formatTypeOptions, diff --git a/forward_engineering/helpers/alterScriptHelpers/common.js b/forward_engineering/helpers/alterScriptHelpers/common.js index 5687749c..6d93f3bc 100644 --- a/forward_engineering/helpers/alterScriptHelpers/common.js +++ b/forward_engineering/helpers/alterScriptHelpers/common.js @@ -1,7 +1,7 @@ const _ = require('lodash'); const { getDiffCopyOptionsByDefault } = require('./tableCopyOptionsHelper'); -const POSSIBLE_UNSET_PROPERTIES = ['DATA_RETENTION_TIME_IN_DAYS', 'MAX_DATA_EXTENSION_TIME_IN_DAYS']; +const POSSIBLE_UNSET_PROPERTIES = ['description', 'DATA_RETENTION_TIME_IN_DAYS', 'MAX_DATA_EXTENSION_TIME_IN_DAYS']; const POSSIBLE_SET_PROPERTIES = [ 'targetLag', 'warehouse', diff --git a/forward_engineering/helpers/alterScriptHelpers/commonScript.js b/forward_engineering/helpers/alterScriptHelpers/commonScript.js index f13df298..8d30976d 100644 --- a/forward_engineering/helpers/alterScriptHelpers/commonScript.js +++ b/forward_engineering/helpers/alterScriptHelpers/commonScript.js @@ -41,7 +41,7 @@ module.exports = ({ getName, getFullName, templates, assignTemplates, tab }) => return { script, data }; }; - const mapKey = { + const mapKeyToKeyword = { description: 'COMMENT', targetLag: 'TARGET_LAG', warehouse: 'WAREHOUSE', @@ -57,7 +57,7 @@ module.exports = ({ getName, getFullName, templates, assignTemplates, tab }) => } else if (key === 'targetLag') { const { targetLagAmount, targetLagType, targetLagDownstream } = data[operation].targetLag?.[0]; - return `${targetLagDownstream ? 'DOWNSTREAM' : `'${targetLagAmount} ${targetLagType}'`}\n`; + return `${targetLagDownstream ? 'DOWNSTREAM' : `'${targetLagAmount} ${targetLagType}'`}`; } return propValue; @@ -74,7 +74,7 @@ module.exports = ({ getName, getFullName, templates, assignTemplates, tab }) => const setPropertyData = Object.keys(setProperty).map((key, index) => { const propValue = setProperty[key]; const value = getValue({ key, data, propValue }, 'setProperty'); - const statement = `${mapKey[key]} = ${value}`; + const statement = `${mapKeyToKeyword[key]} = ${value}`; return Boolean(index) ? tab(statement) : statement; }); @@ -98,7 +98,7 @@ module.exports = ({ getName, getFullName, templates, assignTemplates, tab }) => } const unsetPropertyData = unsetProperty.map((key, index) => { - key = mapKey[key]; + key = mapKeyToKeyword[key]; return Boolean(index) ? `${key}` : `${key}`; }); From c5aed88f071e170de9bdc345522175d394c2ded2 Mon Sep 17 00:00:00 2001 From: yevhenii-moroziuk Date: Wed, 21 Aug 2024 12:25:04 +0300 Subject: [PATCH 5/8] HCK-7642: fix code smells --- forward_engineering/ddlProvider.js | 8 ++++---- .../helpers/alterScriptHelpers/commonScript.js | 4 ++-- forward_engineering/helpers/tableHelper.js | 15 ++++++++------- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/forward_engineering/ddlProvider.js b/forward_engineering/ddlProvider.js index eb69cc3a..c796071d 100644 --- a/forward_engineering/ddlProvider.js +++ b/forward_engineering/ddlProvider.js @@ -78,10 +78,10 @@ module.exports = (baseProvider, options, app) => { }); const getOutOfLineConstraints = ( + isParentActivated, foreignKeyConstraints = [], primaryKeyConstraints = [], uniqueKeyConstraints = [], - isParentActivated, ) => { const constraints = [...foreignKeyConstraints, ...primaryKeyConstraints, ...uniqueKeyConstraints].map( constraint => @@ -325,7 +325,7 @@ module.exports = (baseProvider, options, app) => { isCaseSensitive: tableData.isCaseSensitive, }); - if (!!tableData.dynamic) { + if (tableData.dynamic) { const dynamicTableOptions = getDynamicTableProps({ tableData, tagsStatement, @@ -395,10 +395,10 @@ module.exports = (baseProvider, options, app) => { column_definitions: columnDefinitions, out_of_line_constraints: getOutOfLineConstraints( + isActivated, tableData.foreignKeyConstraints, tableData.compositePrimaryKeys, tableData.compositeUniqueKeys, - isActivated, ), }); } @@ -413,10 +413,10 @@ module.exports = (baseProvider, options, app) => { ), column_definitions: columnDefinitions, out_of_line_constraints: getOutOfLineConstraints( + isActivated, tableData.foreignKeyConstraints, tableData.compositePrimaryKeys, tableData.compositeUniqueKeys, - isActivated, ), }); }, diff --git a/forward_engineering/helpers/alterScriptHelpers/commonScript.js b/forward_engineering/helpers/alterScriptHelpers/commonScript.js index 8d30976d..709e4f84 100644 --- a/forward_engineering/helpers/alterScriptHelpers/commonScript.js +++ b/forward_engineering/helpers/alterScriptHelpers/commonScript.js @@ -55,9 +55,9 @@ module.exports = ({ getName, getFullName, templates, assignTemplates, tab }) => return escapeString(scriptFormat, propValue); } else if (key === 'targetLag') { - const { targetLagAmount, targetLagType, targetLagDownstream } = data[operation].targetLag?.[0]; + const { targetLagAmount, targetLagType, targetLagDownstream } = data[operation].targetLag?.[0] ?? {}; - return `${targetLagDownstream ? 'DOWNSTREAM' : `'${targetLagAmount} ${targetLagType}'`}`; + return targetLagDownstream ? 'DOWNSTREAM' : `'${targetLagAmount} ${targetLagType}'`; } return propValue; diff --git a/forward_engineering/helpers/tableHelper.js b/forward_engineering/helpers/tableHelper.js index b02cda77..8f9e7d03 100644 --- a/forward_engineering/helpers/tableHelper.js +++ b/forward_engineering/helpers/tableHelper.js @@ -51,6 +51,13 @@ module.exports = app => { return `TARGET_LAG = ${targetLagDownstream ? 'DOWNSTREAM' : `'${targetLagAmount} ${targetLagType}'`}\n`; } + function getSelectStatement(selectStatement) { + const mapStatement = (statement, index, statements) => + index === statements.length - 1 ? `\t${statement}` : `\t${statement}\n`; + + return `AS\n${selectStatement.split('\n').map(mapStatement).join('')}`; + } + const getDynamicTableProps = ({ tableData, transient, @@ -67,7 +74,6 @@ module.exports = app => { const { selectStatement } = tableData; const { - iceberg, targetLag, warehouse, refreshMode, @@ -81,12 +87,7 @@ module.exports = app => { return { targetLag: targetLag?.[0] ? getTargetLag(targetLag[0]) : '', warehouse: warehouse ? `WAREHOUSE = ${warehouse}\n` : '', - selectStatement: selectStatement - ? `AS\n${selectStatement - .split('\n') - .map((it, i, arr) => (i === arr.length - 1 ? `\t${it}` : `\t${it}\n`)) - .join('')}` - : '', + selectStatement: selectStatement ? getSelectStatement(selectStatement) : '', externalVolume: externalVolume ? `EXTERNAL_VOLUME = ${externalVolume}\n` : '', catalog: catalog ? `CATALOG = ${catalog}\n` : '', baseLocation: baseLocation ? `BASE_LOCATION = ${baseLocation}\n` : '', From 2cc98a12e3834a97f91a606d52fd4155cca3afe6 Mon Sep 17 00:00:00 2001 From: yevhenii-moroziuk Date: Wed, 21 Aug 2024 17:05:30 +0300 Subject: [PATCH 6/8] HCK-7642: fix second comment --- forward_engineering/helpers/alterScriptHelpers/commonScript.js | 2 +- forward_engineering/helpers/tableHelper.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/forward_engineering/helpers/alterScriptHelpers/commonScript.js b/forward_engineering/helpers/alterScriptHelpers/commonScript.js index 709e4f84..21bf82c3 100644 --- a/forward_engineering/helpers/alterScriptHelpers/commonScript.js +++ b/forward_engineering/helpers/alterScriptHelpers/commonScript.js @@ -55,7 +55,7 @@ module.exports = ({ getName, getFullName, templates, assignTemplates, tab }) => return escapeString(scriptFormat, propValue); } else if (key === 'targetLag') { - const { targetLagAmount, targetLagType, targetLagDownstream } = data[operation].targetLag?.[0] ?? {}; + const { targetLagAmount, targetLagType, targetLagDownstream } = data[operation].targetLag ?? {}; return targetLagDownstream ? 'DOWNSTREAM' : `'${targetLagAmount} ${targetLagType}'`; } diff --git a/forward_engineering/helpers/tableHelper.js b/forward_engineering/helpers/tableHelper.js index 8f9e7d03..b1fc08bb 100644 --- a/forward_engineering/helpers/tableHelper.js +++ b/forward_engineering/helpers/tableHelper.js @@ -85,7 +85,7 @@ module.exports = app => { } = tableData.dynamicTableProps; return { - targetLag: targetLag?.[0] ? getTargetLag(targetLag[0]) : '', + targetLag: targetLag ? getTargetLag(targetLag) : '', warehouse: warehouse ? `WAREHOUSE = ${warehouse}\n` : '', selectStatement: selectStatement ? getSelectStatement(selectStatement) : '', externalVolume: externalVolume ? `EXTERNAL_VOLUME = ${externalVolume}\n` : '', From 230ec7d7956fa5dc69f5fa05cd661205e4083954 Mon Sep 17 00:00:00 2001 From: yevhenii-moroziuk Date: Wed, 21 Aug 2024 17:58:43 +0300 Subject: [PATCH 7/8] HCK-7642: change type to block --- properties_pane/entity_level/entityLevelConfig.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/properties_pane/entity_level/entityLevelConfig.json b/properties_pane/entity_level/entityLevelConfig.json index 2985abd7..35da29d0 100644 --- a/properties_pane/entity_level/entityLevelConfig.json +++ b/properties_pane/entity_level/entityLevelConfig.json @@ -198,7 +198,7 @@ making sure that you maintain a proper JSON format. { "propertyName": "Target Lag", "propertyKeyword": "targetLag", - "propertyType": "group", + "propertyType": "block", "structure": [ { "propertyName": "Type", From db6714c5601a8330230705b8acd86c1b709dd7c6 Mon Sep 17 00:00:00 2001 From: yevhenii-moroziuk Date: Thu, 22 Aug 2024 18:29:33 +0300 Subject: [PATCH 8/8] HCK-7642: fix formatting & target lag --- .../helpers/alterScriptHelpers/commonScript.js | 4 ++-- forward_engineering/helpers/tableHelper.js | 16 ++++++++++------ .../entity_level/entityLevelConfig.json | 14 ++++++++------ 3 files changed, 20 insertions(+), 14 deletions(-) diff --git a/forward_engineering/helpers/alterScriptHelpers/commonScript.js b/forward_engineering/helpers/alterScriptHelpers/commonScript.js index 21bf82c3..767d011d 100644 --- a/forward_engineering/helpers/alterScriptHelpers/commonScript.js +++ b/forward_engineering/helpers/alterScriptHelpers/commonScript.js @@ -55,9 +55,9 @@ module.exports = ({ getName, getFullName, templates, assignTemplates, tab }) => return escapeString(scriptFormat, propValue); } else if (key === 'targetLag') { - const { targetLagAmount, targetLagType, targetLagDownstream } = data[operation].targetLag ?? {}; + const { targetLagAmount, targetLagTimeSpan, targetLagDownstream } = data[operation].targetLag ?? {}; - return targetLagDownstream ? 'DOWNSTREAM' : `'${targetLagAmount} ${targetLagType}'`; + return targetLagDownstream ? 'DOWNSTREAM' : `'${targetLagAmount} ${targetLagTimeSpan}'`; } return propValue; diff --git a/forward_engineering/helpers/tableHelper.js b/forward_engineering/helpers/tableHelper.js index b1fc08bb..7e0b4cff 100644 --- a/forward_engineering/helpers/tableHelper.js +++ b/forward_engineering/helpers/tableHelper.js @@ -47,8 +47,12 @@ module.exports = app => { const mergeKeys = keys => keys.map(key => `"${key.name}"`).join(', '); - function getTargetLag({ targetLagType, targetLagAmount, targetLagDownstream }) { - return `TARGET_LAG = ${targetLagDownstream ? 'DOWNSTREAM' : `'${targetLagAmount} ${targetLagType}'`}\n`; + function getTargetLag({ targetLagTimeSpan, targetLagAmount, targetLagDownstream }) { + if (!(targetLagTimeSpan && targetLagAmount) && !targetLagDownstream) { + return ''; + } + + return `TARGET_LAG = ${targetLagDownstream ? 'DOWNSTREAM' : `'${targetLagAmount} ${targetLagTimeSpan}'`}\n`; } function getSelectStatement(selectStatement) { @@ -85,12 +89,12 @@ module.exports = app => { } = tableData.dynamicTableProps; return { - targetLag: targetLag ? getTargetLag(targetLag) : '', + targetLag: getTargetLag(targetLag), warehouse: warehouse ? `WAREHOUSE = ${warehouse}\n` : '', selectStatement: selectStatement ? getSelectStatement(selectStatement) : '', - externalVolume: externalVolume ? `EXTERNAL_VOLUME = ${externalVolume}\n` : '', - catalog: catalog ? `CATALOG = ${catalog}\n` : '', - baseLocation: baseLocation ? `BASE_LOCATION = ${baseLocation}\n` : '', + externalVolume: externalVolume ? `EXTERNAL_VOLUME = '${externalVolume}'\n` : '', + catalog: catalog ? `CATALOG = '${catalog}'\n` : '', + baseLocation: baseLocation ? `BASE_LOCATION = '${baseLocation}'\n` : '', column_definitions: columnDefinitions ? `\t(\n\t\t${columnDefinitions}\n\t)\n` : '', refreshMode: refreshMode ? `REFRESH_MODE = ${refreshMode}\n` : '', initialize: initialize ? `INITIALIZE = ${initialize}\n` : '', diff --git a/properties_pane/entity_level/entityLevelConfig.json b/properties_pane/entity_level/entityLevelConfig.json index 35da29d0..c580b57e 100644 --- a/properties_pane/entity_level/entityLevelConfig.json +++ b/properties_pane/entity_level/entityLevelConfig.json @@ -5,6 +5,8 @@ * The software may be used and/or copied only with the written permission of * IntegrIT S.A. or in accordance with the terms and conditions stipulated in * the agreement/contract under which the software has been supplied. +* IntegrIT S.A. or in accordance with the terms and conditions stipulated in +* the agreement/contract under which the software has been supplied. In order to define custom properties for any object's properties pane, you may copy/paste from the following, making sure that you maintain a proper JSON format. @@ -70,8 +72,8 @@ making sure that you maintain a proper JSON format. ] }, // “groupInput” can have the following states - 0 items, 1 item, and many items. -// “blockInput” has only 2 states - 0 items or 1 item. -// This gives us an easy way to represent it as an object and not as an array internally which is beneficial for processing +// “blockInput” has only 2 states - 0 items or 1 item. +// This gives us an easy way to represent it as an object and not as an array internally which is beneficial for processing // and forward-engineering in particular. { "propertyName": "Block", @@ -99,7 +101,7 @@ making sure that you maintain a proper JSON format. "propertyKeyword": "keyList", "propertyType": "fieldList", "template": "orderedList" - }, + }, { "propertyName": "List with attribute", "propertyKeyword": "keyListOrder", @@ -155,7 +157,7 @@ making sure that you maintain a proper JSON format. } }, { - "propertyName": "As Dynamic", + "propertyName": "Dynamic", "propertyKeyword": "dynamic", "propertyType": "checkbox" }, @@ -201,8 +203,8 @@ making sure that you maintain a proper JSON format. "propertyType": "block", "structure": [ { - "propertyName": "Type", - "propertyKeyword": "targetLagType", + "propertyName": "Time span", + "propertyKeyword": "targetLagTimeSpan", "propertyType": "select", "options": ["seconds", "minutes", "hours", "days"] },