From ec2aeed9cedd9d1132be3987ba7283741e84cfda Mon Sep 17 00:00:00 2001 From: bbsankarsh Date: Thu, 29 Aug 2024 13:02:49 +0530 Subject: [PATCH 1/2] Enhanced MSSQL metadata collection to include foreign keys and table row counts. --- .../adapters/mssql/mappers/tables.py | 2 +- .../adapters/mssql/models/column.py | 2 + .../adapters/mssql/models/table.py | 2 + .../adapters/mssql/repository.py | 42 +++++++++++++++---- 4 files changed, 40 insertions(+), 8 deletions(-) diff --git a/odd-collector/odd_collector/adapters/mssql/mappers/tables.py b/odd-collector/odd_collector/adapters/mssql/mappers/tables.py index 67de442..ebb35a9 100644 --- a/odd-collector/odd_collector/adapters/mssql/mappers/tables.py +++ b/odd-collector/odd_collector/adapters/mssql/mappers/tables.py @@ -18,7 +18,7 @@ def map_table(table: Table, generator: MssqlGenerator): oddrn = generator.get_oddrn_by_path("tables") map_col = partial(map_column, oddrn_generator=generator, parent_oddrn_path="tables") - dataset = DataSet(field_list=lmap(map_col, table.columns)) + dataset = DataSet(rows_number=table.table_rows,field_list=lmap(map_col, table.columns)) return DataEntity( oddrn=oddrn, diff --git a/odd-collector/odd_collector/adapters/mssql/models/column.py b/odd-collector/odd_collector/adapters/mssql/models/column.py index 513a3d2..c704126 100644 --- a/odd-collector/odd_collector/adapters/mssql/models/column.py +++ b/odd-collector/odd_collector/adapters/mssql/models/column.py @@ -14,6 +14,7 @@ class Column: column_default: str is_nullable: bool is_primary_key: bool + is_foreign_key: bool data_type: str character_maximum_length: int character_octet_length: int @@ -37,3 +38,4 @@ def odd_metadata(self) -> Dict[str, str]: def __post_init__(self): self.is_primary_key = bool(self.is_primary_key) + self.is_foreign_key = bool(self.is_foreign_key) diff --git a/odd-collector/odd_collector/adapters/mssql/models/table.py b/odd-collector/odd_collector/adapters/mssql/models/table.py index d4684d6..f98db9d 100644 --- a/odd-collector/odd_collector/adapters/mssql/models/table.py +++ b/odd-collector/odd_collector/adapters/mssql/models/table.py @@ -1,4 +1,5 @@ from dataclasses import asdict, dataclass, field +from typing import Optional from funcy import omit from odd_collector.adapters.mssql.models.column import Column @@ -11,6 +12,7 @@ class Table: table_schema: str table_name: str table_type: str + table_rows: Optional[int] = None columns: list[Column] = field(default_factory=list) @property diff --git a/odd-collector/odd_collector/adapters/mssql/repository.py b/odd-collector/odd_collector/adapters/mssql/repository.py index 317a988..dda97e7 100644 --- a/odd-collector/odd_collector/adapters/mssql/repository.py +++ b/odd-collector/odd_collector/adapters/mssql/repository.py @@ -26,13 +26,26 @@ def get_columns_for(self, database: str, schema: str, name: str) -> list[Column] TABLES_QUERY: str = """ SELECT - table_catalog, - table_schema, - table_name, - table_type -FROM information_schema.tables -WHERE table_type != 'VIEW' -ORDER BY table_catalog, table_schema, table_name;""" + t.table_catalog, + t.table_schema, + t.table_name, + t.table_type, + SUM(p.rows) AS table_rows +FROM information_schema.tables t +JOIN sys.schemas s ON t.table_schema = s.name +JOIN sys.objects o ON t.table_name = o.name AND o.schema_id = s.schema_id +JOIN sys.partitions p ON o.object_id = p.object_id +WHERE t.table_type = 'BASE TABLE' AND p.index_id IN (0, 1) +GROUP BY + t.table_catalog, + t.table_schema, + t.table_name, + t.table_type +ORDER BY + t.table_catalog, + t.table_schema, + t.table_name; +""" COLUMNS_QUERY: str = """ WITH primary_keys as ( @@ -41,6 +54,12 @@ def get_columns_for(self, database: str, schema: str, name: str) -> list[Column] INNER JOIN information_schema.key_column_usage AS KU ON TC.constraint_name = KU.constraint_name AND TC.constraint_type = 'PRIMARY KEY' +), +foreign_keys AS ( + SELECT CU.table_catalog, CU.table_schema, CU.table_name, CU.column_name + FROM information_schema.referential_constraints AS RC + INNER JOIN information_schema.constraint_column_usage AS CU + ON RC.constraint_name = CU.constraint_name ) SELECT C.table_catalog, @@ -54,6 +73,10 @@ def get_columns_for(self, database: str, schema: str, name: str) -> list[Column] WHEN PK.column_name IS NOT NULL THEN 1 ELSE 0 END AS is_primary_key, + CASE + WHEN FK.column_name IS NOT NULL THEN 1 + ELSE 0 + END AS is_foreign_key, C.data_type, C.character_maximum_length, C.character_octet_length, @@ -76,6 +99,11 @@ def get_columns_for(self, database: str, schema: str, name: str) -> list[Column] AND C.table_schema = PK.table_schema AND C.table_name = PK.table_name AND C.column_name = PK.column_name +LEFT JOIN foreign_keys AS FK + ON C.table_catalog = FK.table_catalog + AND C.table_schema = FK.table_schema + AND C.table_name = FK.table_name + AND C.column_name = FK.column_name ORDER BY C.table_catalog, C.table_schema, C.table_name, C.ordinal_position """ From fa31f50cd8db4a052030308bb846bdeeefad46e2 Mon Sep 17 00:00:00 2001 From: bbsankarsh Date: Thu, 19 Sep 2024 10:14:41 +0530 Subject: [PATCH 2/2] Formated the code with linter --- odd-collector/odd_collector/adapters/mssql/mappers/tables.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/odd-collector/odd_collector/adapters/mssql/mappers/tables.py b/odd-collector/odd_collector/adapters/mssql/mappers/tables.py index ebb35a9..af94926 100644 --- a/odd-collector/odd_collector/adapters/mssql/mappers/tables.py +++ b/odd-collector/odd_collector/adapters/mssql/mappers/tables.py @@ -18,7 +18,9 @@ def map_table(table: Table, generator: MssqlGenerator): oddrn = generator.get_oddrn_by_path("tables") map_col = partial(map_column, oddrn_generator=generator, parent_oddrn_path="tables") - dataset = DataSet(rows_number=table.table_rows,field_list=lmap(map_col, table.columns)) + dataset = DataSet( + rows_number=table.table_rows, field_list=lmap(map_col, table.columns) + ) return DataEntity( oddrn=oddrn,