release 10.627.24135

.github/workflows/build.yml

@@ -1,4 +1,4 @@
-# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# This workflow will install Python dependencies, run tests, and lint with a variety of Python versions
 # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
 
 name: build
@@ -7,7 +7,7 @@ on:
   push:
     branches: [ "master", "dev" ]
     tags:
-      - '[0-9]*'
+      - '[0-9]+'
 
 jobs:
   test:
@@ -20,10 +20,12 @@ jobs:
 
     steps:
     - uses: actions/checkout@v3
+
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v3
       with:
         python-version: ${{ matrix.python-version }}
+
     - name: Cache pip
       uses: actions/cache@v2
       with:
@@ -32,32 +34,51 @@ jobs:
         restore-keys: |
           ${{ runner.os }}-pip-${{ matrix.python-version }}-
           ${{ runner.os }}-pip-
-    - name: Install dependencies
+
+    - name: Install Dependencies with Retry
       run: |
         cp packaging/setup.py .
-        pip install .
+        max_attempts=5
+        sleep_seconds=10
+        attempt_num=1
+        until pip install .; do
+          echo "Attempt $attempt_num of $max_attempts failed! Trying again in $sleep_seconds seconds..."
+          sleep $sleep_seconds
+          ((attempt_num++))
+          if [[ $attempt_num -eq $max_attempts ]]; then
+            echo "All $max_attempts attempts have failed!"
+            exit 1
+          fi
+        done
+
+    - name: Install Test Dependencies
+      run: |
         python tests/smoke_test.py
         python -m pip install --upgrade pip
         pip install --no-deps -r tests/requirements.lock
         pip install .
+
     - name: Run Tests
+      env:
+        CI: true
       run: |
         python -m unittest discover tests -v
 
   coverage:
     if: github.ref == 'refs/heads/dev'
     runs-on: ubuntu-latest
     strategy:
-      fail-fast: false
       matrix:
         python-version: ["3.12"]
 
     steps:
     - uses: actions/checkout@v3
+
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v3
       with:
         python-version: ${{ matrix.python-version }}
+
     - name: Cache pip
       uses: actions/cache@v2
       with:
@@ -66,6 +87,23 @@ jobs:
         restore-keys: |
           ${{ runner.os }}-pip-${{ matrix.python-version }}-
           ${{ runner.os }}-pip-
+
+    - name: Install Dependencies with Retry
+      run: |
+        cp packaging/setup.py .
+        max_attempts=5
+        sleep_seconds=10
+        attempt_num=1
+        until pip install .; do
+          echo "Attempt $attempt_num of $max_attempts failed! Trying again in $sleep_seconds seconds..."
+          sleep $sleep_seconds
+          ((attempt_num++))
+          if [[ $attempt_num -eq $max_attempts ]]; then
+            echo "All $max_attempts attempts have failed!"
+            exit 1
+          fi
+        done
+
     - name: Coverage
       env:
         COVERAGE: true
@@ -74,7 +112,6 @@ jobs:
         python -m pip install --upgrade pip
         pip install --no-deps -r tests/requirements.lock
         pip install coverage coveralls
-        cp packaging/setup.py .
         pip install .
         coverage run --rcfile=packaging/coverage.ini -m unittest discover tests
         coverage report --rcfile=packaging/coverage.ini

.gitignore

@@ -17,3 +17,4 @@ vendor/
 /setup.py
 
 
+/MANIFEST.in

mo_sql_parsing/formatting.py

@@ -113,6 +113,7 @@ def isolate(expr, sql, prec):
     "where",
     "groupby",
     "having",
+    "union_all"
 ]
 
 ordered_clauses = [
@@ -589,7 +590,7 @@ def unordered_query(self, json, prec):
             part
             for clause in unordered_clauses
             if clause in json
-            for part in [getattr(self, clause)(json, precedence["from"])]
+            for part in [getattr(self, clause)(json, precedence[clause]+1)]
             if part
         )
         if prec > precedence["from"]:
@@ -598,12 +599,13 @@ def unordered_query(self, json, prec):
             return f"({sql})"
 
     def with_(self, json, prec):
-        if "with" in json:
-            with_ = json["with"]
-            if not isinstance(with_, list):
-                with_ = [with_]
-            parts = ", ".join("{0} AS ({1})".format(part["name"], self.dispatch(part["value"])) for part in with_)
-            return "WITH {0}".format(parts)
+        with_ = listwrap(json["with"])
+        parts = ", ".join(f"{part['name']} AS (\n{self.dispatch(part['value'])}\n)" for part in with_)
+        return f"WITH {parts}"
+
+    def union_all(self, json, prec):
+        sql = "\nUNION ALL\n".join(self.dispatch(part) for part in listwrap(json['union_all']))
+        return f"{sql}" if prec > precedence["union_all"] else f"({sql})"
 
     def select(self, json, prec):
         select = json["select"]

mo_sql_parsing/keywords.py

@@ -331,9 +331,15 @@
     "join": 18,
     "list": 18,
     "case": 19,
+    "with": 30,
     "select": 30,
-    "from": 30,
+    "select_distinct": 30,
+    "distinct_on": 31,
+    "from": 32,
+    "where": 33,
+    "groupby": 34,
     "window": 35,
+    "having": 35,
     "union": 40,
     "union_all": 40,
     "except": 40,

mo_sql_parsing/sql_parser.py

@@ -27,7 +27,7 @@ def mysql_parser(all_columns):
     utils.emit_warning_for_double_quotes = False
 
     mysql_string = regex_string | ansi_string | mysql_doublequote_string
-    atomic_ident = mysql_backtick_ident | sqlserver_ident | ident_w_dash
+    atomic_ident = mysql_backtick_ident | sqlserver_ident | ident_w_dash/no_dashes
     return parser(mysql_string, atomic_ident, all_columns=all_columns)
 
 
@@ -38,7 +38,7 @@ def sqlserver_parser(all_columns):
 
 def bigquery_parser(all_columns):
     mysql_string = regex_string | ansi_string | mysql_doublequote_string
-    atomic_ident = ansi_ident | mysql_backtick_ident | simple_ident
+    atomic_ident = ansi_ident | mysql_backtick_ident | ident_w_dash
     return parser(mysql_string, atomic_ident, all_columns=all_columns)
 
 
@@ -690,6 +690,7 @@ def mult(tokens):
                 assign("engine", EQ + identifier)
                 | assign("collate", EQ + identifier)
                 | assign("auto_increment", EQ + int_num)
+                | assign("autoincrement", EQ + int_num)
                 | assign("comment", EQ + literal_string)
                 | assign("default character set", EQ + identifier)
                 | assign("default charset", EQ + identifier)

mo_sql_parsing/types.py

@@ -255,6 +255,7 @@ def get_column_type(expr, identifier, literal_string):
         | (NULL / True)("nullable")
         | flag("unique")
         | flag("auto_increment")
+        | flag("autoincrement")
         | assign("comment", literal_string)
         | assign("character set", identifier)
         | assign("collate", Optional(EQ) + identifier)

mo_sql_parsing/utils.py

@@ -890,6 +890,6 @@ def no_dashes(tokens, start, string):
 digit = Char("0123456789")
 with whitespaces.NO_WHITESPACE:
     ident_w_dash = Char(FIRST_IDENT_CHAR) + (Regex("(?<=[^ 0-9])\\-(?=[^ 0-9])") | Char(IDENT_CHAR))[...]
-    ident_w_dash = Regex(ident_w_dash.__regex__()[1]).set_parser_name("identifier_with_dashes") / no_dashes
+    ident_w_dash = Regex(ident_w_dash.__regex__()[1]).set_parser_name("identifier_with_dashes")
 
 simple_ident = Word(FIRST_IDENT_CHAR, IDENT_CHAR).set_parser_name("identifier")

packaging/requirements.txt

@@ -1,4 +1,4 @@
-mo-future
-mo-dots
-mo-parsing
-mo-imports
+mo-future==7.546.24057
+mo-dots==9.578.24081
+mo-parsing==8.581.24094
+mo-imports==7.546.24057

packaging/setup.py

@@ -6,7 +6,7 @@
     author_email='[email protected]',
     classifiers=["Development Status :: 5 - Production/Stable","Topic :: Software Development :: Libraries","Topic :: Software Development :: Libraries :: Python Modules","Programming Language :: SQL","Programming Language :: Python :: 3.8","Programming Language :: Python :: 3.9","License :: OSI Approved :: Mozilla Public License 2.0 (MPL 2.0)","Programming Language :: Python :: 3.10","Programming Language :: Python :: 3.11","Programming Language :: Python :: 3.12"],
     description='More SQL Parsing! Parse SQL into JSON parse tree',
-    extras_require={"dev":[],"tests":["mo-testing","mo-threads","mo-files","mo-streams","zstandard"]},
+    extras_require={"dev":[],"tests":["mo-testing==7.562.24075","mo-threads==6.570.24076","mo-files==6.570.24076","mo-streams==1.570.24076","zstandard>=0.22.0"]},
     include_package_data=True,
     install_requires=["mo-dots==9.578.24081","mo-future==7.546.24057","mo-imports==7.546.24057","mo-parsing==8.581.24094"],
     license='MPL 2.0',
@@ -15,6 +15,6 @@
     name='mo-sql-parsing',
     packages=["mo_sql_parsing"],
     url='https://github.com/klahnakoski/mo-sql-parsing',
-    version='10.581.24094',
+    version='10.627.24135',
     zip_safe=True
 )

packaging/setuptools.json

@@ -16,7 +16,10 @@
     "description": "More SQL Parsing! Parse SQL into JSON parse tree",
     "extras_require": {
         "dev": [],
-        "tests": ["mo-testing", "mo-threads", "mo-files", "mo-streams", "zstandard"]
+        "tests": [
+            "mo-testing==7.562.24075", "mo-threads==6.570.24076",   "mo-files==6.570.24076",
+            "mo-streams==1.570.24076",       "zstandard>=0.22.0"
+        ]
     },
     "include_package_data": true,
     "install_requires": [
@@ -311,6 +314,6 @@
     "name": "mo-sql-parsing",
     "packages": ["mo_sql_parsing"],
     "url": "https://github.com/klahnakoski/mo-sql-parsing",
-    "version": "10.581.24094",
+    "version": "10.627.24135",
     "zip_safe": true
 }

tests/requirements.lock

@@ -3,7 +3,7 @@
 hjson==3.1.0
 mo-collections==5.556.24070
 mo-dots==9.578.24081
-mo-files==6.556.24070
+mo-files==6.562.24075
 mo-future==7.546.24057
 mo-imports==7.546.24057
 mo-json==6.556.24070
@@ -13,7 +13,7 @@ mo-math==7.552.24062
 mo-parsing==8.581.24094
 mo-sql-parsing==10.581.24094
 mo-streams==1.556.24070
-mo-testing==7.559.24071
-mo-threads==6.556.24070
+mo-testing==7.562.24075
+mo-threads==6.562.24075
 mo-times==5.556.24070
 zstandard==0.22.0

tests/requirements.txt

@@ -1,5 +1,5 @@
-mo-testing>=7.559.24071
-mo-threads>=6.556.24070
-mo-files>=6.556.24070
-mo-streams>=1.556.24070
-zstandard>=0.22.0
+mo-testing==7.562.24075
+mo-threads==6.570.24076
+mo-files==6.570.24076
+mo-streams==1.570.24076
+zstandard>=0.22.0

tests/test_bigquery.py

@@ -1749,3 +1749,9 @@ def test_issue_224(self):
         }
 
         self.assertEqual(result, expected)
+
+    def test_issue_227_name(self):
+        query = """ select * from proj-prd.dataset.table"""
+        result = parse(query)
+        expected = {"from": "proj-prd.dataset.table", "select": {"all_columns": {}}}
+        self.assertEqual(result, expected)

tests/test_format_and_parse.py

@@ -11,9 +11,8 @@
 import re
 from unittest import TestCase
 
-from mo_future import get_function_name
 from mo_logs import logger
-from mo_parsing.debug import Debugger
+from mo_testing.fuzzytestcase import add_error_reporting
 
 from mo_sql_parsing import format, parse
 from mo_sql_parsing.keywords import join_keywords
@@ -29,6 +28,7 @@
 """
 
 
+@add_error_reporting
 def remove_whitespace(sql):
     # WE ASSUME A WHITESPACE REMOVAL IS GOOD ENOUGH FOR COMPARE
     return re.sub(r"\s+", "", sql, flags=re.UNICODE)

tests/test_formatting.py

@@ -313,7 +313,7 @@ def test_issue_104(self):
         self.assertEqual(result, expected)
 
     def test_with_cte(self):
-        expected = "WITH t AS (SELECT a FROM table) SELECT * FROM t"
+        expected = "WITH t AS (\nSELECT a FROM table\n) SELECT * FROM t"
         result = format({
             "select": {"all_columns": {}},
             "from": "t",
@@ -322,7 +322,7 @@ def test_with_cte(self):
         self.assertEqual(result, expected)
 
     def test_with_cte_various(self):
-        expected = "WITH t1 AS (SELECT a FROM table), t2 AS (SELECT 1) SELECT * FROM t1, t2"
+        expected = "WITH t1 AS (\nSELECT a FROM table\n), t2 AS (\nSELECT 1\n) SELECT * FROM t1, t2"
         result = format({
             "select": {"all_columns": {}},
             "from": ["t1", "t2"],
@@ -359,7 +359,7 @@ def test_issue_34_intersect(self):
         self.assertEqual(format_result, query)
 
     def test_issue_34_union_all(self):
-        query = "SELECT stuid FROM student UNION ALL SELECT stuid FROM student"
+        query = "SELECT stuid FROM student\nUNION ALL\nSELECT stuid FROM student"
         parse_result = parse(query)
         format_result = format(parse_result)
         self.assertEqual(format_result, query)
@@ -831,4 +831,9 @@ def test_issue_217(self):
     def test_issue_220(self):
         sql = """SELECT TO_TIMESTAMP(A DEFAULT NULL ON CONVERSION ERROR, 'DD/MM/YYYY HH24:MI:SS') FROM B.C"""
         result = format(parse(sql))
+        self.assertEqual(result, sql)
+
+    def test_issue_233_format_union(self):
+        sql = """WITH table_test AS (\nSELECT public.categories.string_agg AS string_agg FROM public.categories\n) SELECT * FROM table_test\nUNION ALL\nSELECT * FROM table_test"""
+        result = format(parse(sql))
         self.assertEqual(result, sql)