Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

HCK-8896: Code improvements #242

Merged
merged 4 commits into from
Dec 5, 2024
Merged

HCK-8896: Code improvements #242

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
50f02de
HCK-8896: Move view script creation to the provider
Nightlngale Nov 29, 2024
8646e27
HCK-8896: Refactor functions
Nightlngale Nov 29, 2024
bc962ef
HCK-8896: fix script
Nightlngale Nov 29, 2024
ddc19da
HCK-8896: fix script
Nightlngale Nov 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions forward_engineering/alterScript/alterScriptFromDeltaHelper.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -228,8 +228,11 @@ const getAlterViewsScriptDtos = (schema, provider, _, dbVersion) => {
.map(view => ({ ...view, ...(view.role || {}) }))
.filter(view => !view.compMod?.created && !view.compMod?.deleted)
.flatMap(getScript);

const addedViewScriptDtos = getViewScripts(getItems(schema, 'views', 'added'), 'created', getAddViewsScripts(_));
const addedViewScriptDtos = getViewScripts(
getItems(schema, 'views', 'added'),
'created',
getAddViewsScripts(provider, _),
);
const deletedViewScriptDtos = getViewScripts(
getItems(schema, 'views', 'deleted'),
'deleted',
Expand Down
8 changes: 4 additions & 4 deletions forward_engineering/alterScript/alterScriptHelpers/alterViewHelper.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
const { getViewScript } = require('../../helpers/viewHelper');
const { hydrateTableProperties } = require('../../helpers/tableHelper');
const {
getEntityData,
Expand Down Expand Up @@ -87,9 +86,10 @@ const hydrateAlterView = _ => view => {
/**
* @return {(view: Object) => AlterScriptDto}
* */
const getAddViewsScripts = _ => view => {
const getAddViewsScripts = (provider, _) => view => {
const hydratedView = hydrateView(_)(view);
const script = getViewScript({ _, ...hydratedView });
const script = provider.createView(hydratedView);

return {
isActivated: true,
scripts: [
Expand Down Expand Up @@ -137,7 +137,7 @@ const getModifyViewsScripts = (provider, _, dbVersion) => view => {
}
const dropViewScript = provider.dropView(viewName);
const hydratedView = hydrateView(_)(view);
const addViewScript = getViewScript({ _, ...hydratedView });
const addViewScript = provider.createView(hydratedView);
const dropViewScriptDto = AlterScriptDto.getInstance([dropViewScript], true, true);
const addViewScriptDto = AlterScriptDto.getInstance([addViewScript], true, false);

Expand Down
6 changes: 2 additions & 4 deletions forward_engineering/api.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
'use strict';

const { getDatabaseStatement, getUseCatalogStatement } = require('./helpers/databaseHelper');
const { getViewScript } = require('./helpers/viewHelper');
const { getCleanedUrl, buildScript, isSupportUnityCatalog, getDBVersionNumber } = require('./utils/general');
const fetchRequestHelper = require('../reverse_engineering/helpers/fetchRequestHelper');
const databricksHelper = require('../reverse_engineering/helpers/databricksHelper');
Expand Down Expand Up @@ -259,16 +258,15 @@ module.exports = {
* */
generateViewScript(data, logger, callback, app) {
try {
const _ = app.require('lodash');
const provider = require('./ddlProvider/ddlProvider')(app);
const viewSchema = JSON.parse(data.jsonSchema || '{}');
const dbVersion = data.modelData[0].dbVersion;
const isUnityCatalogSupports = isSupportUnityCatalog(dbVersion);

const useCatalogStatement = isUnityCatalogSupports ? getUseCatalogStatement(data.containerData) : '';
const databaseStatement = getDatabaseStatement(data.containerData, isUnityCatalogSupports, dbVersion);

const script = getViewScript({
_,
const script = provider.createView({
schema: viewSchema,
viewData: data.viewData,
containerData: data.containerData,
Expand Down
65 changes: 64 additions & 1 deletion forward_engineering/ddlProvider/ddlProvider.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,76 @@
const templates = require('./ddlTemplates');
const { getFullEntityName } = require('../utils/general');
const {
getFullEntityName,
getName,
replaceSpaceWithUnderscore,
prepareName,
getContainerName,
getEntityData,
getEntityProperties,
wrapInBrackets,
} = require('../utils/general');
const { getViewTagsStatement } = require('../helpers/unityTagsHelper');
const { getTablePropertiesClause } = require('../helpers/tableHelper');
const viewHelper = require('../helpers/viewHelper');

module.exports = app => {
const { assignTemplates } = app.require('@hackolade/ddl-fe-utils');
const _ = app.require('lodash');
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@Nightlngale ideally getting deps like this should be migrated to direct deps in the plugin when possible :)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@bigorn0 I couldn’t agree more. But here I was trying to touch a small part of code related only to "Create view" script

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@Nightlngale that's true in general that we want to limit the impacted surface but for such tech debt and given how often we go into plugins code I think the opposite: If we don't do our sanity/cleanup changes along the way when we are adapting plugins then we NEVER get the chance to actually tackle such debt :(

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I agree with Ugo it's a good opportunity to fix it. But I would propose fixing it in a separate PR. @Nightlngale could you please create PR with this tech improvement?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@bigorn0 @VitaliiBedletskyi yes, I will add this improvement in scope of the separate PR


return {
dropView(name) {
return assignTemplates(templates.dropView, { name });
},

createView(data) {
const { schema, viewData, containerData, collectionRefsDefinitionsMap } = data;

const columns = schema.properties || {};
const view = _.first(viewData) || {};

if (!view.isActivated) {
return;
}

const bucketName = replaceSpaceWithUnderscore(prepareName(viewHelper.retrieveContainerName(containerData)));
const viewName = replaceSpaceWithUnderscore(prepareName(view.code || view.name));
const isGlobal = schema.viewGlobal && schema.viewTemporary;
const isTemporary = schema.viewTemporary;
const orReplace = schema.viewOrReplace;
const ifNotExists = view.viewIfNotExist;
const name = bucketName ? `${bucketName}.${viewName}` : `${viewName}`;
const tableProperties =
schema.tableProperties && Array.isArray(schema.tableProperties)
? viewHelper.filterRedundantProperties(schema.tableProperties, ['transient_lastDdlTime'])
: [];
const viewUnityTagsStatements =
schema.unityViewTags && getViewTagsStatement({ viewSchema: schema, viewName: name });

return assignTemplates(templates.createView, {
orReplace: orReplace && !ifNotExists ? ' OR REPLACE' : '',
global: isGlobal ? ' GLOBAL' : '',
temporary: isTemporary ? ' TEMPORARY' : '',
ifNotExists: ifNotExists ? ' IF NOT EXISTS' : '',
name,
columnList: view.columnList
? `${wrapInBrackets(view.columnList)}`
: viewHelper.getDefaultColumnList(columns),
schemaBinding: '',
comment: viewHelper.getCommentStatement(schema.description),
tablePropertyStatements: tableProperties.length
? `TBLPROPERTIES (${getTablePropertiesClause(_)(tableProperties)})`
: '',
query: schema.selectStatement
? `AS ${schema.selectStatement}`
: viewHelper.getTableSelectStatement({
_,
collectionRefsDefinitionsMap,
columns,
}),
viewUnityTagsStatements: viewUnityTagsStatements ? `${viewUnityTagsStatements};` : '',
});
},

dropTableIndex(name) {
return name ? assignTemplates(templates.dropTableIndex, { name }) : '';
},
Expand Down
3 changes: 2 additions & 1 deletion forward_engineering/ddlProvider/ddlTemplates.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
module.exports = {
createView:
'CREATE ${orReplace}${global}${temporary} VIEW${ifNotExists} ${name}${columnList}${schemaBinding}${comment}${tablePropertyStatements}${query};${viewUnityTagsStatements}',
'CREATE${orReplace}${global}${temporary} VIEW${ifNotExists} ${name} ${columnList}${schemaBinding}${comment}${tablePropertyStatements}${query};${viewUnityTagsStatements}',

dropView: 'DROP VIEW IF EXISTS ${name};',

alterViewName: 'ALTER VIEW ${oldName} RENAME TO ${newName};',
Expand Down
8 changes: 4 additions & 4 deletions forward_engineering/helpers/feScriptBuilder.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,13 +131,12 @@ const buildEntityLevelFEScript =
* @param _ {any}
* @return {Array<ContainerLevelEntityDto>}
* */
const getContainerLevelViewScriptDtos = (data, _) => {
const getContainerLevelViewScriptDtos = (data, provider, _) => {
return data.views
.map(viewId => {
const viewSchema = JSON.parse(data.jsonSchema[viewId] || '{}');
const viewData = data.viewData[viewId];
const viewScript = getViewScript({
_,
const viewScript = provider.createView({
schema: viewSchema,
viewData: viewData,
containerData: data.containerData,
Expand Down Expand Up @@ -295,8 +294,9 @@ const buildContainerLevelFEScriptDto =
const arePkFkConstraintsAvailable = isSupportUnityCatalog(dbVersion);
const areNotNullConstraintsAvailable = isSupportNotNullConstraints(dbVersion);

const provider = require('../ddlProvider/ddlProvider')(app);
const useCatalogStatement = arePkFkConstraintsAvailable ? getUseCatalogStatement(containerData) : '';
const viewsScriptDtos = getContainerLevelViewScriptDtos(data, _);
const viewsScriptDtos = getContainerLevelViewScriptDtos(data, provider, _);
const databaseStatement = getDatabaseStatement(containerData, arePkFkConstraintsAvailable, dbVersion);
const entityScriptDtos = await getContainerLevelEntitiesScriptDtos(
app,
Expand Down
59 changes: 8 additions & 51 deletions forward_engineering/helpers/viewHelper.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,6 @@
'use strict';

const { prepareName, encodeStringLiteral, commentDeactivatedStatement } = require('../utils/general');
const { getTablePropertiesClause } = require('./tableHelper');
const { getViewTagsStatement } = require('./unityTagsHelper');
const ddlTemplates = require('../ddlProvider/ddlTemplates');
const assignTemplates = require('../utils/assignTemplates');

const getColumnNames = _ => (collectionRefsDefinitionsMap, columns) => {
return _.uniq(
Expand Down Expand Up @@ -121,57 +117,18 @@ function getTableSelectStatement({ _, collectionRefsDefinitionsMap, columns }) {
return '';
}

module.exports = {
getViewScript({ _, schema, viewData, containerData, collectionRefsDefinitionsMap }) {
const columns = schema.properties || {};
const view = _.first(viewData) || {};

if (!view.isActivated) {
return;
}

const bucketName = replaceSpaceWithUnderscore(prepareName(retrieveContainerName(containerData)));
const viewName = replaceSpaceWithUnderscore(prepareName(view.code || view.name));
const isGlobal = schema.viewGlobal && schema.viewTemporary;
const isTemporary = schema.viewTemporary;
const orReplace = schema.viewOrReplace;
const ifNotExists = view.viewIfNotExist;
const name = bucketName ? `${bucketName}.${viewName}` : `${viewName}`;
const tableProperties =
schema.tableProperties && Array.isArray(schema.tableProperties)
? filterRedundantProperties(schema.tableProperties, ['transient_lastDdlTime'])
: [];
const viewUnityTagsStatements =
schema.unityViewTags && getViewTagsStatement({ viewSchema: schema, viewName: name });

return assignTemplates(ddlTemplates.createView, {
orReplace: orReplace && !ifNotExists ? 'OR REPLACE ' : '',
global: isGlobal ? 'GLOBAL ' : '',
temporary: isTemporary ? 'TEMPORARY ' : '',
ifNotExists: ifNotExists ? ' IF NOT EXISTS' : '',
name,
columnList: view.columnList ? `\n(${view.columnList})` : getDefaultColumnList(columns),
schemaBinding: '',
comment: getCommentStatement(schema.description),
tablePropertyStatements: tableProperties.length
? `\nTBLPROPERTIES (${getTablePropertiesClause(_)(tableProperties)})`
: '',
query: schema.selectStatement
? `\nAS ${schema.selectStatement}`
: getTableSelectStatement({
_,
collectionRefsDefinitionsMap,
columns,
}),
viewUnityTagsStatements: viewUnityTagsStatements ? `\n${viewUnityTagsStatements};` : '',
});
},
};

const filterRedundantProperties = (tableProperties, propertiesList) => {
if (!Array.isArray(tableProperties)) {
return tableProperties;
}

return tableProperties.filter(prop => !propertiesList.includes(prop.propertyKey));
};

module.exports = {
getTableSelectStatement,
retrieveContainerName,
filterRedundantProperties,
getDefaultColumnList,
getCommentStatement,
};
15 changes: 0 additions & 15 deletions forward_engineering/utils/assignTemplates.js
View file
Open in desktop

This file was deleted.

2 changes: 1 addition & 1 deletion forward_engineering/utils/general.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,7 @@ const wrapInTicks = (str = '') => {
};

const wrapInBrackets = (str = '') => {
return /^\(\S+\)$/.test(str) ? str : `(${str})`;
return /^\(.*\)$/.test(str) ? str : `(${str})`;
};

/**
Expand Down
Loading