Hck 7898 quick win deltalake support new runtime 15 (#235)

* Hck 7915 add runtime 15 to the list of available versions and make it the default (#229) * HCK-7915: added Runtime 15 option * HCK-7915: added Runtime 15 to model config * HCK-7916: default config for variant data type (#228) * HCK-7916: default config for variant data type * HCK-7931: fix commenting out of DROP statements with a checkbox (#226) * HCK-7916: added Variant version bound --------- Co-authored-by: Vitalii Yarmus <[email protected]> * HCK-7919: add Variant type to RE from file (Hive) grammar (#232) * HCK-7898: added config for variant synonym (#233) * HCK-7920: implemented variant subtype resolve logic on REing from ins… (#234) * HCK-7920: implemented variant subtype resolve logic on REing from instance * HCK-7920: moved columns with json detection logic to separate function * HCK-7920: fixed remarks * HCK-7920: changed variant type order in config * HCK-7920: fixed json types ordering * HCK-7920: covered empty list case * HCK-7898: removed variant from being default * HCK-7898: removed redundant config --------- Co-authored-by: Vitalii Yarmus <[email protected]>
hackolade · Sep 13, 2024 · f60c7e7 · f60c7e7
1 parent b0b51ac
commit f60c7e7
Show file tree

Hide file tree

Showing 19 changed files with 10,118 additions and 9,516 deletions.
diff --git a/package.json b/package.json
@@ -19,7 +19,8 @@
                 "Runtime 11",
                 "Runtime 12",
                 "Runtime 13",
-                "Runtime 14"
+                "Runtime 14",
+                "Runtime 15"
             ]
         },
         "features": {

diff --git a/properties_pane/defaultData.json b/properties_pane/defaultData.json
@@ -1,7 +1,7 @@
 {
 	"model": {
 		"modelName": "New model",
-		"dbVersion": "Runtime 14",
+		"dbVersion": "Runtime 15",
 		"dbVendor": "Delta Lake"
 	},
 	"container": {

diff --git a/properties_pane/field_level/fieldLevelConfig.json b/properties_pane/field_level/fieldLevelConfig.json
@@ -2930,6 +2930,301 @@ making sure that you maintain a proper JSON format.
 					]
 				}
 			}
+		],
+		"variant": [
+			"name",
+			"code",
+			"sampleName",
+			"schemaId",
+			"refPath",
+			{
+				"propertyName": "Synonym",
+				"propertyKeyword": "mode",
+				"propertyType": "select",
+				"options": ["var"],
+				"hidden": true
+			},
+			{
+				"propertyName": "Comments",
+				"propertyKeyword": "description",
+				"propertyTooltip": "comments",
+				"addTimestampButton": false,
+				"propertyType": "details",
+				"template": "textarea"
+			},
+			"type",
+			{
+				"propertyName": "Variant type",
+				"propertyKeyword": "variantType",
+				"shouldValidate": false,
+				"propertyTooltip": "Select from list of options",
+				"propertyType": "select",
+				"options": ["JSON"],
+				"dependency": {
+					"key": "childType",
+					"value": "variant"
+				}
+			},
+			{
+				"propertyName": "Subtype",
+				"propertyKeyword": "subtype",
+				"propertyType": "select",
+				"options": [
+					{ "name": "object", "value": "object" },
+					{ "name": "array", "value": "array" },
+					{ "name": "string", "value": "string" },
+					{ "name": "number", "value": "number" },
+					{ "name": "boolean", "value": "boolean" },
+					{ "name": "null", "value": "null" }
+				],
+				"dependency": {
+					"key": "childType",
+					"value": "variant"
+				}
+			},
+			{
+				"propertyName": "Not null",
+				"propertyKeyword": "required",
+				"enableForReference": true,
+				"propertyType": "checkbox"
+			},
+			{
+				"propertyName": "Primary key",
+				"propertyKeyword": "compositePrimaryKey",
+				"propertyType": "checkbox",
+				"propertyTooltip": {
+					"disabled": [
+						{
+							"tooltip": "This column is part of the table composite primary key definition. Please refer to this definition if you want more information or to update the Primary Key definition",
+							"dependency": {
+								"key": "compositePrimaryKey",
+								"value": true
+							}
+						},
+						{
+							"tooltip": "Remove the existing composite primary key definition prior to unlock the possibility to mark this single column as the new primary key for this table",
+							"dependency": {
+								"level": "siblings",
+								"key": "compositePrimaryKey",
+								"value": true
+							}
+						},
+						{
+							"tooltip": "Another column has already been selected as primary key.  You must unselect it prior to either selecting this column, or creating a compound primary key.",
+							"dependency": {
+								"level": "siblings",
+								"key": "primaryKey",
+								"value": true
+							}
+						}
+					]
+				},
+				"dependency": {
+					"type": "or",
+					"values": [
+						{
+							"key": "compositePrimaryKey",
+							"value": true
+						},
+						{
+							"level": "siblings",
+							"key": "primaryKey",
+							"value": true
+						}
+					]
+				},
+				"disabled": true
+			},
+			{
+				"propertyName": "Primary key",
+				"propertyKeyword": "primaryKey",
+				"preserveOnCleanDependencies": true,
+				"enableForReference": true,
+				"propertyType": "checkbox",
+				"dependency": {
+					"type": "and",
+					"values": [
+						{
+							"type": "not",
+							"values": {
+								"level": "siblings",
+								"key": "primaryKey",
+								"value": true
+							}
+						},
+						{
+							"type": "or",
+							"values": [
+								{
+									"key": "compositePrimaryKey",
+									"value": false
+								},
+								{
+									"key": "compositePrimaryKey",
+									"exist": false
+								}
+							]
+						}
+					]
+				}
+			},
+			{
+				"propertyName": "Unique",
+				"propertyKeyword": "compositeUniqueKey",
+				"propertyType": "checkbox",
+				"dependency": {
+					"key": "compositeUniqueKey",
+					"value": true
+				},
+				"disabled": true
+			},
+			{
+				"propertyName": "Unique",
+				"propertyKeyword": "unique",
+				"enableForReference": true,
+				"propertyType": "checkbox",
+				"dependency": {
+					"type": "and",
+					"values": [
+						{
+							"type": "or",
+							"values": [
+								{
+									"key": "compositePrimaryKey",
+									"value": false
+								},
+								{
+									"key": "compositePrimaryKey",
+									"exist": false
+								}
+							]
+						},
+						{
+							"type": "or",
+							"values": [
+								{
+									"key": "compositeUniqueKey",
+									"value": false
+								},
+								{
+									"key": "compositeUniqueKey",
+									"exist": false
+								}
+							]
+						}
+					]
+				}
+			},
+			{
+				"propertyName": "Check constraint",
+				"propertyKeyword": "check",
+				"propertyType": "details",
+				"template": "textarea",
+				"markdown": false
+			},
+			{
+				"propertyName": "Check constraint name",
+				"propertyKeyword": "checkConstraintName",
+				"propertyType": "text"
+			},
+			{
+				"propertyName": "Generated as",
+				"propertyTooltip": "",
+				"propertyKeyword": "generatedDefaultValue",
+				"propertyType": "block",
+				"structure": [
+					{
+						"propertyName": "Type",
+						"propertyKeyword": "generatedType",
+						"propertyType": "select",
+						"options": ["always", "by default"],
+						"defaultValue": "always"
+					},
+					{
+						"propertyName": "Expression",
+						"propertyKeyword": "expression",
+						"propertyType": "details",
+						"template": "textarea",
+						"markdown": false
+					}
+				]
+			},
+			"additionalProperties",
+			{
+				"propertyName": "Remarks",
+				"propertyKeyword": "comments",
+				"shouldValidate": false,
+				"propertyTooltip": "remarks",
+				"addTimestampButton": true,
+				"propertyType": "details",
+				"template": "textarea"
+			},
+			{
+				"propertyName": "Column Tags",
+				"propertyKeyword": "unityColumnTags",
+				"propertyTooltip": "Contains all the tags that have been applied to the column. Available only in Unity Catalog",
+				"propertyType": "group",
+				"groupItemLimit": 20,
+				"structure": [
+					{
+						"propertyName": "Key",
+						"propertyKeyword": "unityTagKey",
+						"propertyTooltip": "The key of the tag must be between 1 and 255 UTF-8 characters inclusive, special characters cannot be used in tag names",
+						"propertyType": "text",
+						"validation": {
+							"regex": "^(?=.*?\\d)?(?=.*?[a-zA-Z])?[a-zA-Z\\d]{1,255}$"
+						}
+					},
+					{
+						"propertyName": "Value",
+						"propertyKeyword": "unityTagValue",
+						"propertyTooltip": "The value of the tag.",
+						"propertyType": "details",
+						"template": "textarea",
+						"markdown": false
+					}
+				],
+				"dependency": {
+					"type": "not",
+					"values": [
+						{
+							"level": "model",
+							"key": "dbVersion",
+							"value": "Runtime 6"
+						},
+						{
+							"level": "model",
+							"key": "dbVersion",
+							"value": "Runtime 7"
+						},
+						{
+							"level": "model",
+							"key": "dbVersion",
+							"value": "Runtime 8"
+						},
+						{
+							"level": "model",
+							"key": "dbVersion",
+							"value": "Runtime 9"
+						},
+						{
+							"level": "model",
+							"key": "dbVersion",
+							"value": "Runtime 10"
+						},
+						{
+							"level": "model",
+							"key": "dbVersion",
+							"value": "Runtime 11"
+						},
+						{
+							"level": "model",
+							"key": "dbVersion",
+							"value": "Runtime 12"
+						}
+					]
+				}
+			}
 		]
 	}
 }
diff --git a/properties_pane/model_level/modelLevelConfig.json b/properties_pane/model_level/modelLevelConfig.json
@@ -142,7 +142,8 @@ making sure that you maintain a proper JSON format.
 					"Runtime 11",
 					"Runtime 12",
 					"Runtime 13",
-					"Runtime 14"
+					"Runtime 14",
+					"Runtime 15"
 				]
 			},
 			{

diff --git a/reverse_engineering/api.js b/reverse_engineering/api.js
@@ -22,6 +22,8 @@ const { parseDDLStatements } = require('./parseDDLStatements');
 const { isSupportUnityCatalog } = require('./helpers/databricksHelper');
 const unityTagsHelper = require('./helpers/unityTagsHelper');
 const { adaptJsonSchema } = require('./adaptJsonSchema');
+const { getVariantColumnsWithResolvedSubtype } = require('./helpers/variantPropertiesSubtypeResolveHelper');
+const { getPropertiesPotentiallyContainingJSON } = require('./helpers/columnsREHelper');
 
 const DEFAULT_DATABRICKS_CATALOG_NAME = 'hive_metastore';
 
@@ -265,12 +267,12 @@ module.exports = {
 							logger,
 						);
 
-						const columnsOfTypeString = (tableData.properties || []).filter(
-							property => property.mode === 'string',
+						const columnsPotentiallyContainingJSON = getPropertiesPotentiallyContainingJSON(
+							tableData.properties,
 						);
-						const hasColumnsOfTypeString = !dependencies.lodash.isEmpty(columnsOfTypeString);
+						const hasPotentiallyContainingJSONColumns = columnsPotentiallyContainingJSON.length > 0;
 						let documents = [];
-						if (hasColumnsOfTypeString) {
+						if (hasPotentiallyContainingJSONColumns) {
 							progress({
 								message: 'Start getting documents from table',
 								containerName: 'databases',
@@ -280,10 +282,15 @@ module.exports = {
 								connectionInfo: connectionData,
 								dbName,
 								tableName: table.name,
-								fields: columnsOfTypeString,
+								fields: columnsPotentiallyContainingJSON,
 								recordSamplingSettings: data.recordSamplingSettings,
 								logger,
 							});
+							tableData.schema = getVariantColumnsWithResolvedSubtype({
+								propertiesSchema: tableData.schema,
+								documents,
+								logger,
+							});
 							progress({
 								message: 'Documents retrieved successfully',
 								containerName: 'databases',

diff --git a/reverse_engineering/grammars/HiveLexer.g4 b/reverse_engineering/grammars/HiveLexer.g4
@@ -121,6 +121,7 @@ KW_VARCHAR: V A R C H A R;
 KW_ARRAY: A R R A Y;
 KW_STRUCT: S T R U C T;
 KW_MAP: M A P;
+KW_VARIANT: V A R I A N T;
 KW_UNIONTYPE: U N I O N T Y P E;
 KW_REDUCE: R E D U C E;
 KW_PARTITIONED: P A R T I T I O N E D;

diff --git a/reverse_engineering/grammars/HiveParser.g4 b/reverse_engineering/grammars/HiveParser.g4
@@ -1330,6 +1330,7 @@ primitiveType
     | KW_DECIMAL (LPAREN Number (COMMA Number)? RPAREN)?
     | KW_VARCHAR LPAREN Number RPAREN
     | KW_CHAR LPAREN Number RPAREN
+    | KW_VARIANT
     ;
 
 listType