diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1d05828..f63b6a8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,4 +1,4 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# This workflow will install Python dependencies, run tests, and lint with a variety of Python versions # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python name: build @@ -7,7 +7,7 @@ on: push: branches: [ "master", "dev" ] tags: - - '[0-9]*' + - '[0-9]+' jobs: test: @@ -20,10 +20,12 @@ jobs: steps: - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} + - name: Cache pip uses: actions/cache@v2 with: @@ -32,15 +34,33 @@ jobs: restore-keys: | ${{ runner.os }}-pip-${{ matrix.python-version }}- ${{ runner.os }}-pip- - - name: Install dependencies + + - name: Install Dependencies with Retry run: | cp packaging/setup.py . - pip install . + max_attempts=5 + sleep_seconds=10 + attempt_num=1 + until pip install .; do + echo "Attempt $attempt_num of $max_attempts failed! Trying again in $sleep_seconds seconds..." + sleep $sleep_seconds + ((attempt_num++)) + if [[ $attempt_num -eq $max_attempts ]]; then + echo "All $max_attempts attempts have failed!" + exit 1 + fi + done + + - name: Install Test Dependencies + run: | python tests/smoke_test.py python -m pip install --upgrade pip pip install --no-deps -r tests/requirements.lock pip install . + - name: Run Tests + env: + CI: true run: | python -m unittest discover tests -v @@ -48,16 +68,17 @@ jobs: if: github.ref == 'refs/heads/dev' runs-on: ubuntu-latest strategy: - fail-fast: false matrix: python-version: ["3.12"] steps: - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} + - name: Cache pip uses: actions/cache@v2 with: @@ -66,6 +87,23 @@ jobs: restore-keys: | ${{ runner.os }}-pip-${{ matrix.python-version }}- ${{ runner.os }}-pip- + + - name: Install Dependencies with Retry + run: | + cp packaging/setup.py . + max_attempts=5 + sleep_seconds=10 + attempt_num=1 + until pip install .; do + echo "Attempt $attempt_num of $max_attempts failed! Trying again in $sleep_seconds seconds..." + sleep $sleep_seconds + ((attempt_num++)) + if [[ $attempt_num -eq $max_attempts ]]; then + echo "All $max_attempts attempts have failed!" + exit 1 + fi + done + - name: Coverage env: COVERAGE: true @@ -74,7 +112,6 @@ jobs: python -m pip install --upgrade pip pip install --no-deps -r tests/requirements.lock pip install coverage coveralls - cp packaging/setup.py . pip install . coverage run --rcfile=packaging/coverage.ini -m unittest discover tests coverage report --rcfile=packaging/coverage.ini diff --git a/.gitignore b/.gitignore index 28639ff..893bb8f 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,4 @@ vendor/ /setup.py +/MANIFEST.in diff --git a/mo_sql_parsing/formatting.py b/mo_sql_parsing/formatting.py index 29110bf..45e124f 100644 --- a/mo_sql_parsing/formatting.py +++ b/mo_sql_parsing/formatting.py @@ -113,6 +113,7 @@ def isolate(expr, sql, prec): "where", "groupby", "having", + "union_all" ] ordered_clauses = [ @@ -589,7 +590,7 @@ def unordered_query(self, json, prec): part for clause in unordered_clauses if clause in json - for part in [getattr(self, clause)(json, precedence["from"])] + for part in [getattr(self, clause)(json, precedence[clause]+1)] if part ) if prec > precedence["from"]: @@ -598,12 +599,13 @@ def unordered_query(self, json, prec): return f"({sql})" def with_(self, json, prec): - if "with" in json: - with_ = json["with"] - if not isinstance(with_, list): - with_ = [with_] - parts = ", ".join("{0} AS ({1})".format(part["name"], self.dispatch(part["value"])) for part in with_) - return "WITH {0}".format(parts) + with_ = listwrap(json["with"]) + parts = ", ".join(f"{part['name']} AS (\n{self.dispatch(part['value'])}\n)" for part in with_) + return f"WITH {parts}" + + def union_all(self, json, prec): + sql = "\nUNION ALL\n".join(self.dispatch(part) for part in listwrap(json['union_all'])) + return f"{sql}" if prec > precedence["union_all"] else f"({sql})" def select(self, json, prec): select = json["select"] diff --git a/mo_sql_parsing/keywords.py b/mo_sql_parsing/keywords.py index 83c67ba..fc8d20e 100644 --- a/mo_sql_parsing/keywords.py +++ b/mo_sql_parsing/keywords.py @@ -331,9 +331,15 @@ "join": 18, "list": 18, "case": 19, + "with": 30, "select": 30, - "from": 30, + "select_distinct": 30, + "distinct_on": 31, + "from": 32, + "where": 33, + "groupby": 34, "window": 35, + "having": 35, "union": 40, "union_all": 40, "except": 40, diff --git a/mo_sql_parsing/sql_parser.py b/mo_sql_parsing/sql_parser.py index c33dbee..45adf0d 100644 --- a/mo_sql_parsing/sql_parser.py +++ b/mo_sql_parsing/sql_parser.py @@ -27,7 +27,7 @@ def mysql_parser(all_columns): utils.emit_warning_for_double_quotes = False mysql_string = regex_string | ansi_string | mysql_doublequote_string - atomic_ident = mysql_backtick_ident | sqlserver_ident | ident_w_dash + atomic_ident = mysql_backtick_ident | sqlserver_ident | ident_w_dash/no_dashes return parser(mysql_string, atomic_ident, all_columns=all_columns) @@ -38,7 +38,7 @@ def sqlserver_parser(all_columns): def bigquery_parser(all_columns): mysql_string = regex_string | ansi_string | mysql_doublequote_string - atomic_ident = ansi_ident | mysql_backtick_ident | simple_ident + atomic_ident = ansi_ident | mysql_backtick_ident | ident_w_dash return parser(mysql_string, atomic_ident, all_columns=all_columns) @@ -690,6 +690,7 @@ def mult(tokens): assign("engine", EQ + identifier) | assign("collate", EQ + identifier) | assign("auto_increment", EQ + int_num) + | assign("autoincrement", EQ + int_num) | assign("comment", EQ + literal_string) | assign("default character set", EQ + identifier) | assign("default charset", EQ + identifier) diff --git a/mo_sql_parsing/types.py b/mo_sql_parsing/types.py index faa8ac5..410867b 100644 --- a/mo_sql_parsing/types.py +++ b/mo_sql_parsing/types.py @@ -255,6 +255,7 @@ def get_column_type(expr, identifier, literal_string): | (NULL / True)("nullable") | flag("unique") | flag("auto_increment") + | flag("autoincrement") | assign("comment", literal_string) | assign("character set", identifier) | assign("collate", Optional(EQ) + identifier) diff --git a/mo_sql_parsing/utils.py b/mo_sql_parsing/utils.py index f79f242..3d6336f 100644 --- a/mo_sql_parsing/utils.py +++ b/mo_sql_parsing/utils.py @@ -890,6 +890,6 @@ def no_dashes(tokens, start, string): digit = Char("0123456789") with whitespaces.NO_WHITESPACE: ident_w_dash = Char(FIRST_IDENT_CHAR) + (Regex("(?<=[^ 0-9])\\-(?=[^ 0-9])") | Char(IDENT_CHAR))[...] - ident_w_dash = Regex(ident_w_dash.__regex__()[1]).set_parser_name("identifier_with_dashes") / no_dashes + ident_w_dash = Regex(ident_w_dash.__regex__()[1]).set_parser_name("identifier_with_dashes") simple_ident = Word(FIRST_IDENT_CHAR, IDENT_CHAR).set_parser_name("identifier") diff --git a/packaging/requirements.txt b/packaging/requirements.txt index bf253f0..78eff89 100644 --- a/packaging/requirements.txt +++ b/packaging/requirements.txt @@ -1,4 +1,4 @@ -mo-future -mo-dots -mo-parsing -mo-imports +mo-future==7.546.24057 +mo-dots==9.578.24081 +mo-parsing==8.581.24094 +mo-imports==7.546.24057 diff --git a/packaging/setup.py b/packaging/setup.py index efee8b6..3267477 100644 --- a/packaging/setup.py +++ b/packaging/setup.py @@ -6,7 +6,7 @@ author_email='kyle@lahnakoski.com', classifiers=["Development Status :: 5 - Production/Stable","Topic :: Software Development :: Libraries","Topic :: Software Development :: Libraries :: Python Modules","Programming Language :: SQL","Programming Language :: Python :: 3.8","Programming Language :: Python :: 3.9","License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)","Programming Language :: Python :: 3.10","Programming Language :: Python :: 3.11","Programming Language :: Python :: 3.12"], description='More SQL Parsing! Parse SQL into JSON parse tree', - extras_require={"dev":[],"tests":["mo-testing","mo-threads","mo-files","mo-streams","zstandard"]}, + extras_require={"dev":[],"tests":["mo-testing==7.562.24075","mo-threads==6.570.24076","mo-files==6.570.24076","mo-streams==1.570.24076","zstandard>=0.22.0"]}, include_package_data=True, install_requires=["mo-dots==9.578.24081","mo-future==7.546.24057","mo-imports==7.546.24057","mo-parsing==8.581.24094"], license='MPL 2.0', @@ -15,6 +15,6 @@ name='mo-sql-parsing', packages=["mo_sql_parsing"], url='https://github.com/klahnakoski/mo-sql-parsing', - version='10.581.24094', + version='10.627.24135', zip_safe=True ) \ No newline at end of file diff --git a/packaging/setuptools.json b/packaging/setuptools.json index 8d52037..2eafb93 100644 --- a/packaging/setuptools.json +++ b/packaging/setuptools.json @@ -16,7 +16,10 @@ "description": "More SQL Parsing! Parse SQL into JSON parse tree", "extras_require": { "dev": [], - "tests": ["mo-testing", "mo-threads", "mo-files", "mo-streams", "zstandard"] + "tests": [ + "mo-testing==7.562.24075", "mo-threads==6.570.24076", "mo-files==6.570.24076", + "mo-streams==1.570.24076", "zstandard>=0.22.0" + ] }, "include_package_data": true, "install_requires": [ @@ -311,6 +314,6 @@ "name": "mo-sql-parsing", "packages": ["mo_sql_parsing"], "url": "https://github.com/klahnakoski/mo-sql-parsing", - "version": "10.581.24094", + "version": "10.627.24135", "zip_safe": true } \ No newline at end of file diff --git a/tests/requirements.lock b/tests/requirements.lock index fd95fdb..75e70fd 100644 --- a/tests/requirements.lock +++ b/tests/requirements.lock @@ -3,7 +3,7 @@ hjson==3.1.0 mo-collections==5.556.24070 mo-dots==9.578.24081 -mo-files==6.556.24070 +mo-files==6.562.24075 mo-future==7.546.24057 mo-imports==7.546.24057 mo-json==6.556.24070 @@ -13,7 +13,7 @@ mo-math==7.552.24062 mo-parsing==8.581.24094 mo-sql-parsing==10.581.24094 mo-streams==1.556.24070 -mo-testing==7.559.24071 -mo-threads==6.556.24070 +mo-testing==7.562.24075 +mo-threads==6.562.24075 mo-times==5.556.24070 zstandard==0.22.0 \ No newline at end of file diff --git a/tests/requirements.txt b/tests/requirements.txt index c9e22ea..c87d550 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,5 +1,5 @@ -mo-testing>=7.559.24071 -mo-threads>=6.556.24070 -mo-files>=6.556.24070 -mo-streams>=1.556.24070 -zstandard>=0.22.0 \ No newline at end of file +mo-testing==7.562.24075 +mo-threads==6.570.24076 +mo-files==6.570.24076 +mo-streams==1.570.24076 +zstandard>=0.22.0 diff --git a/tests/test_bigquery.py b/tests/test_bigquery.py index cebddbb..82e807a 100644 --- a/tests/test_bigquery.py +++ b/tests/test_bigquery.py @@ -1749,3 +1749,9 @@ def test_issue_224(self): } self.assertEqual(result, expected) + + def test_issue_227_name(self): + query = """ select * from proj-prd.dataset.table""" + result = parse(query) + expected = {"from": "proj-prd.dataset.table", "select": {"all_columns": {}}} + self.assertEqual(result, expected) diff --git a/tests/test_format_and_parse.py b/tests/test_format_and_parse.py index ce39257..132cb31 100644 --- a/tests/test_format_and_parse.py +++ b/tests/test_format_and_parse.py @@ -11,9 +11,8 @@ import re from unittest import TestCase -from mo_future import get_function_name from mo_logs import logger -from mo_parsing.debug import Debugger +from mo_testing.fuzzytestcase import add_error_reporting from mo_sql_parsing import format, parse from mo_sql_parsing.keywords import join_keywords @@ -29,6 +28,7 @@ """ +@add_error_reporting def remove_whitespace(sql): # WE ASSUME A WHITESPACE REMOVAL IS GOOD ENOUGH FOR COMPARE return re.sub(r"\s+", "", sql, flags=re.UNICODE) diff --git a/tests/test_formatting.py b/tests/test_formatting.py index 96b718c..edf9477 100644 --- a/tests/test_formatting.py +++ b/tests/test_formatting.py @@ -313,7 +313,7 @@ def test_issue_104(self): self.assertEqual(result, expected) def test_with_cte(self): - expected = "WITH t AS (SELECT a FROM table) SELECT * FROM t" + expected = "WITH t AS (\nSELECT a FROM table\n) SELECT * FROM t" result = format({ "select": {"all_columns": {}}, "from": "t", @@ -322,7 +322,7 @@ def test_with_cte(self): self.assertEqual(result, expected) def test_with_cte_various(self): - expected = "WITH t1 AS (SELECT a FROM table), t2 AS (SELECT 1) SELECT * FROM t1, t2" + expected = "WITH t1 AS (\nSELECT a FROM table\n), t2 AS (\nSELECT 1\n) SELECT * FROM t1, t2" result = format({ "select": {"all_columns": {}}, "from": ["t1", "t2"], @@ -359,7 +359,7 @@ def test_issue_34_intersect(self): self.assertEqual(format_result, query) def test_issue_34_union_all(self): - query = "SELECT stuid FROM student UNION ALL SELECT stuid FROM student" + query = "SELECT stuid FROM student\nUNION ALL\nSELECT stuid FROM student" parse_result = parse(query) format_result = format(parse_result) self.assertEqual(format_result, query) @@ -831,4 +831,9 @@ def test_issue_217(self): def test_issue_220(self): sql = """SELECT TO_TIMESTAMP(A DEFAULT NULL ON CONVERSION ERROR, 'DD/MM/YYYY HH24:MI:SS') FROM B.C""" result = format(parse(sql)) + self.assertEqual(result, sql) + + def test_issue_233_format_union(self): + sql = """WITH table_test AS (\nSELECT public.categories.string_agg AS string_agg FROM public.categories\n) SELECT * FROM table_test\nUNION ALL\nSELECT * FROM table_test""" + result = format(parse(sql)) self.assertEqual(result, sql) \ No newline at end of file diff --git a/tests/test_sqlite.py b/tests/test_sqlite.py new file mode 100644 index 0000000..d1ea635 --- /dev/null +++ b/tests/test_sqlite.py @@ -0,0 +1,37 @@ +# encoding: utf-8 +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this file, +# You can obtain one at http://mozilla.org/MPL/2.0/. +# + + +from unittest import TestCase +from mo_sql_parsing import parse + + +class TestSqlite(TestCase): + def test_autoincrement_in_create(self): + sql = """ + CREATE TABLE Products ( + product_id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + price REAL NOT NULL, + description TEXT + ); + """ + result = parse(sql) + + self.assertEqual( + result, + { + 'create table': { + 'name': 'Products', + 'columns': [ + {'name': 'product_id', 'type': {'integer': {}}, 'primary_key': True, 'autoincrement': True}, + {'name': 'name', 'type': {'text': {}}, 'nullable': False}, + {'name': 'price', 'type': {'real': {}}, 'nullable': False}, + {'name': 'description', 'type': {'text': {}}} + ] + } + }, + )