[Bug] Make the ruamel integeration compatiable with legacy versions (#…

…925)

* Rearrange tests

Signed-off-by: Ching Yi, Chan <[email protected]>

* Remove type hints

Signed-off-by: Ching Yi, Chan <[email protected]>

* minify test matrix

Signed-off-by: Ching Yi, Chan <[email protected]>

---------

Signed-off-by: Ching Yi, Chan <[email protected]>

.github/workflows/tests.yaml

@@ -15,22 +15,15 @@ on:
 jobs:
   build:
     runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        version: [ '3.9', '3.10', '3.11' ]
-        sqlalchemy: [ "sqlalchemy>=1.4,<2.0", "sqlalchemy>=2.0" ]
-
     steps:
       - uses: actions/checkout@v3
       - name: Set up Python
         uses: actions/setup-python@v2
         with:
-          python-version: ${{ matrix.version }}
+          python-version: 3.11
       - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install "${{ matrix.sqlalchemy }}"
           if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
           pip install -e '.[duckdb]'
       - name: Run tests

.github/workflows/tox.yaml

@@ -0,0 +1,37 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: Run tox tests
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.10"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+          pip install -e '.[duckdb]'
+          pip install tox
+      - name: Run tox-sqlalchemy
+        run: tox -c tox-sqlalchemy.ini
+      - name: Run tox-ruamel
+        run: tox -c tox-ruamel.ini
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@v3
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

piperider_cli/yaml/__init__.py

@@ -1,8 +1,6 @@
-from pathlib import Path
-from typing import Any, Callable, Optional, Union
+from typing import Any, Callable
 
 from ruamel import yaml
-from ruamel.yaml import StreamTextType, StreamType, VersionType
 from ruamel.yaml import CommentedMap as _cm, CommentedSeq as _cs
 
 _yaml = yaml.YAML()
@@ -12,7 +10,7 @@
 YAMLError = yaml.YAMLError
 
 
-def load(stream: Union[Path, StreamTextType]) -> Any:
+def load(stream) -> Any:
     return _yaml.load(stream)
 
 
@@ -22,12 +20,12 @@ def allow_duplicate_keys_loader() -> Callable:
     return yml.load
 
 
-def safe_load(stream: StreamTextType, version: Optional[VersionType] = None) -> Any:
+def safe_load(stream, version=None) -> Any:
     return yaml.safe_load(stream, version)
 
 
 def dump(
-        data: Union[Path, StreamType], stream: Any = None, *, transform: Any = None
+        data, stream: Any = None, *, transform: Any = None
 ) -> Any:
     return _yaml.dump(data, stream, transform=transform)
 
@@ -56,5 +54,5 @@ def round_trip_load_yaml(file_path):
 
 def round_trip_dump(
         data: Any,
-        stream: Optional[StreamType] = None):
+        stream=None):
     return yaml.round_trip_dump(data, stream)

setup.py

@@ -76,6 +76,7 @@ def _get_version():
           'csv': duckdb_require_packages,
           'parquet': duckdb_require_packages,
           'dev': [
+              'tox',
               'pytest>=4.6',
               'pytest-flake8',
               'flake8==3.9.2',

tox-ruamel.ini

@@ -0,0 +1,25 @@
+[tox]
+env_list = ruamel-{0.17.21,0.17.22}
+minversion = 4.7.0
+
+[testenv]
+description = run the tests with pytest
+package = wheel
+wheel_build_env = .pkg
+
+
+deps =
+    pytest>=6
+    dbt-core>=1.6,<1.7
+    dbt-duckdb>=1.6,<1.7
+    ruamel-0.17.21: ruamel.yaml<=0.17.21
+    ruamel-0.17.22: ruamel.yaml>0.17.21,<0.18
+
+
+commands =
+    pytest {tty:--color=yes} {posargs}
+
+setenv =
+    PYTHONPATH = {toxinidir}/tests/:{env:PYTHONPATH}
+
+

tox-sqlalchemy.ini

@@ -0,0 +1,25 @@
+[tox]
+env_list = sqlalchemy-{14,20}
+minversion = 4.7.0
+
+[testenv]
+description = run the tests with pytest
+package = wheel
+wheel_build_env = .pkg
+
+
+deps =
+    pytest>=6
+    dbt-core>=1.6,<1.7
+    dbt-duckdb>=1.6,<1.7
+    sqlalchemy-14: sqlalchemy>=1.4,<2.0
+    sqlalchemy-20: sqlalchemy>=2.0
+
+
+commands =
+    pytest {tty:--color=yes} {posargs}
+
+setenv =
+    PYTHONPATH = {toxinidir}/tests/:{env:PYTHONPATH}
+
+