From c3acb8ce4df567c1417eb9903d3e33d235f4373a Mon Sep 17 00:00:00 2001 From: Leonid Kurhanskyi Date: Sat, 13 Jul 2024 00:29:51 +0300 Subject: [PATCH 1/3] fix sqlfluff nested configs --- sqllineage/cli.py | 3 ++- sqllineage/core/parser/sqlfluff/analyzer.py | 9 +++++---- sqllineage/runner.py | 7 ++++++- sqllineage/utils/helpers.py | 8 +++++--- 4 files changed, 18 insertions(+), 9 deletions(-) diff --git a/sqllineage/cli.py b/sqllineage/cli.py index 857de5cf..44de2989 100644 --- a/sqllineage/cli.py +++ b/sqllineage/cli.py @@ -107,9 +107,10 @@ def main(args=None) -> None: if args.e and args.f: warnings.warn("Both -e and -f options are specified. -e option will be ignored") if args.f or args.e: - sql = extract_sql_from_args(args) + sql, file_path = extract_sql_from_args(args) runner = LineageRunner( sql, + file_path=file_path, dialect=args.dialect, metadata_provider=metadata_provider, verbose=args.verbose, diff --git a/sqllineage/core/parser/sqlfluff/analyzer.py b/sqllineage/core/parser/sqlfluff/analyzer.py index 74e0ace8..93e9aabe 100644 --- a/sqllineage/core/parser/sqlfluff/analyzer.py +++ b/sqllineage/core/parser/sqlfluff/analyzer.py @@ -27,10 +27,13 @@ class SqlFluffLineageAnalyzer(LineageAnalyzer): PARSER_NAME = "sqlfluff" SUPPORTED_DIALECTS = list(dialect.label for dialect in dialect_readout()) - def __init__(self, dialect: str, silent_mode: bool = False): + def __init__(self, file_path: str, dialect: str, silent_mode: bool = False): self._dialect = dialect self._silent_mode = silent_mode self.tsql_split_cache: Dict[str, BaseSegment] = {} + self.sqlfluff_config = FluffConfig.from_path( + path=file_path, overrides={"dialect": self._dialect} + ) def split_tsql(self, sql: str) -> List[str]: """ @@ -79,9 +82,7 @@ def analyze( ) def _list_specific_statement_segment(self, sql: str): - parsed = Linter( - config=FluffConfig.from_root(overrides={"dialect": self._dialect}) - ).parse_string(sql) + parsed = Linter(config=self.sqlfluff_config).parse_string(sql) violations = [ str(e) for e in parsed.violations diff --git a/sqllineage/runner.py b/sqllineage/runner.py index 9d666b53..784f8db2 100644 --- a/sqllineage/runner.py +++ b/sqllineage/runner.py @@ -37,6 +37,7 @@ class LineageRunner(object): def __init__( self, sql: str, + file_path: str = ".", dialect: str = DEFAULT_DIALECT, metadata_provider: MetaDataProvider = DummyMetaDataProvider(), verbose: bool = False, @@ -47,6 +48,7 @@ def __init__( The entry point of SQLLineage after command line options are parsed. :param sql: a string representation of SQL statements. + :param file_path: path of the SQL file. :param dialect: sql dialect :param metadata_provider: metadata service object providing table schema :param verbose: verbose flag indicating whether statement-wise lineage result will be shown @@ -60,6 +62,7 @@ def __init__( stacklevel=2, ) self._sql = sql + self._file_path = file_path self._verbose = verbose self._draw_options = draw_options if draw_options else {} self._evaluated = False @@ -183,7 +186,9 @@ def _eval(self): analyzer = ( SqlParseLineageAnalyzer() if self._dialect == SQLPARSE_DIALECT - else SqlFluffLineageAnalyzer(self._dialect, self._silent_mode) + else SqlFluffLineageAnalyzer( + self._file_path, self._dialect, self._silent_mode + ) ) if SQLLineageConfig.TSQL_NO_SEMICOLON and self._dialect == "tsql": self._stmt = analyzer.split_tsql(self._sql.strip()) diff --git a/sqllineage/utils/helpers.py b/sqllineage/utils/helpers.py index df9c3ede..2e2ec2d4 100644 --- a/sqllineage/utils/helpers.py +++ b/sqllineage/utils/helpers.py @@ -1,6 +1,6 @@ import logging from argparse import Namespace -from typing import List +from typing import List, Tuple logger = logging.getLogger(__name__) @@ -25,11 +25,13 @@ def escape_identifier_name(name: str): return name.lower() -def extract_sql_from_args(args: Namespace) -> str: +def extract_sql_from_args(args: Namespace) -> Tuple[str, str]: sql = "" + file_path = "." if getattr(args, "f", None): try: with open(args.f) as f: + file_path = args.f sql = f.read() except IsADirectoryError: logger.exception("%s is a directory", args.f) @@ -43,7 +45,7 @@ def extract_sql_from_args(args: Namespace) -> str: exit(1) elif getattr(args, "e", None): sql = args.e - return sql + return sql, file_path def split(sql: str) -> List[str]: From e6e3d8e06f1a2d7925ae4e75400912d6f5d1a580 Mon Sep 17 00:00:00 2001 From: Leonid Kurhanskyi Date: Sat, 13 Jul 2024 01:35:25 +0300 Subject: [PATCH 2/3] fix tests --- sqllineage/cli.py | 5 +++-- sqllineage/runner.py | 4 ++-- sqllineage/utils/helpers.py | 15 ++++++++++----- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/sqllineage/cli.py b/sqllineage/cli.py index 44de2989..f72c0310 100644 --- a/sqllineage/cli.py +++ b/sqllineage/cli.py @@ -16,7 +16,7 @@ from sqllineage.drawing import draw_lineage_graph from sqllineage.runner import LineageRunner from sqllineage.utils.constant import LineageLevel -from sqllineage.utils.helpers import extract_sql_from_args +from sqllineage.utils.helpers import extract_sql_from_args, extract_file_path_from_args logger = logging.getLogger(__name__) @@ -107,7 +107,8 @@ def main(args=None) -> None: if args.e and args.f: warnings.warn("Both -e and -f options are specified. -e option will be ignored") if args.f or args.e: - sql, file_path = extract_sql_from_args(args) + sql = extract_sql_from_args(args) + file_path = extract_file_path_from_args(args) runner = LineageRunner( sql, file_path=file_path, diff --git a/sqllineage/runner.py b/sqllineage/runner.py index 784f8db2..113d0360 100644 --- a/sqllineage/runner.py +++ b/sqllineage/runner.py @@ -37,22 +37,22 @@ class LineageRunner(object): def __init__( self, sql: str, - file_path: str = ".", dialect: str = DEFAULT_DIALECT, metadata_provider: MetaDataProvider = DummyMetaDataProvider(), verbose: bool = False, silent_mode: bool = False, draw_options: Optional[Dict[str, Any]] = None, + file_path: str = ".", ): """ The entry point of SQLLineage after command line options are parsed. :param sql: a string representation of SQL statements. - :param file_path: path of the SQL file. :param dialect: sql dialect :param metadata_provider: metadata service object providing table schema :param verbose: verbose flag indicating whether statement-wise lineage result will be shown :param silent_mode: boolean flag indicating whether to skip lineage analysis for unknown statement types + :param file_path: path of the SQL file. """ if dialect == SQLPARSE_DIALECT: warnings.warn( diff --git a/sqllineage/utils/helpers.py b/sqllineage/utils/helpers.py index 2e2ec2d4..86fbbdab 100644 --- a/sqllineage/utils/helpers.py +++ b/sqllineage/utils/helpers.py @@ -1,6 +1,6 @@ import logging from argparse import Namespace -from typing import List, Tuple +from typing import List logger = logging.getLogger(__name__) @@ -25,13 +25,11 @@ def escape_identifier_name(name: str): return name.lower() -def extract_sql_from_args(args: Namespace) -> Tuple[str, str]: +def extract_sql_from_args(args: Namespace) -> str: sql = "" - file_path = "." if getattr(args, "f", None): try: with open(args.f) as f: - file_path = args.f sql = f.read() except IsADirectoryError: logger.exception("%s is a directory", args.f) @@ -45,7 +43,14 @@ def extract_sql_from_args(args: Namespace) -> Tuple[str, str]: exit(1) elif getattr(args, "e", None): sql = args.e - return sql, file_path + return sql + + +def extract_file_path_from_args(args: Namespace) -> str: + file_path = "." + if getattr(args, "f", None): + file_path = args.f + return file_path def split(sql: str) -> List[str]: From c17cbac59972234bf2c7b7a06846a447a544f9a7 Mon Sep 17 00:00:00 2001 From: Leonid Kurhanskyi Date: Sat, 13 Jul 2024 01:44:34 +0300 Subject: [PATCH 3/3] fix isort --- sqllineage/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sqllineage/cli.py b/sqllineage/cli.py index f72c0310..e4630a0d 100644 --- a/sqllineage/cli.py +++ b/sqllineage/cli.py @@ -16,7 +16,7 @@ from sqllineage.drawing import draw_lineage_graph from sqllineage.runner import LineageRunner from sqllineage.utils.constant import LineageLevel -from sqllineage.utils.helpers import extract_sql_from_args, extract_file_path_from_args +from sqllineage.utils.helpers import extract_file_path_from_args, extract_sql_from_args logger = logging.getLogger(__name__)