diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3f798602b..9f9dbd2c4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -267,74 +267,75 @@ jobs: AIRFLOW_CONN_DATABRICKS_DEFAULT: ${{ secrets.AIRFLOW_CONN_DATABRICKS_DEFAULT }} DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }} - Run-Integration-Tests-Sqlite: - needs: Authorize - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.11"] - airflow-version: ["2.8"] - - steps: - - uses: actions/checkout@v3 - with: - ref: ${{ github.event.pull_request.head.sha || github.ref }} - - uses: actions/cache@v3 - with: - path: | - ~/.cache/pip - .local/share/hatch/ - key: integration-sqlite-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('cosmos/__init__.py') }} - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python-version }} - - - name: Install packages and dependencies - run: | - python -m pip install uv - uv pip install --system hatch - hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }} run pip freeze - - - name: Test Cosmos against Airflow ${{ matrix.airflow-version }} and Python ${{ matrix.python-version }} - run: | - hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration-sqlite-setup - hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration-sqlite - env: - AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ - AIRFLOW_CONN_EXAMPLE_CONN: postgres://postgres:postgres@0.0.0.0:5432/postgres - AIRFLOW_CONN_AWS_S3_CONN: ${{ secrets.AIRFLOW_CONN_AWS_S3_CONN }} - AIRFLOW_CONN_GCP_GS_CONN: ${{ secrets.AIRFLOW_CONN_GCP_GS_CONN }} - AIRFLOW_CONN_AZURE_ABFS_CONN: ${{ secrets.AIRFLOW_CONN_AZURE_ABFS_CONN }} - AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 90.0 - PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH - AIRFLOW__COSMOS__ENABLE_CACHE: 0 - COSMOS_CONN_POSTGRES_PASSWORD: ${{ secrets.COSMOS_CONN_POSTGRES_PASSWORD }} - DATABRICKS_CLUSTER_ID: mock - DATABRICKS_HOST: mock - DATABRICKS_WAREHOUSE_ID: mock - DATABRICKS_TOKEN: mock - POSTGRES_HOST: localhost - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: postgres - POSTGRES_SCHEMA: public - POSTGRES_PORT: 5432 - AIRFLOW__COSMOS__REMOTE_TARGET_PATH: "s3://cosmos-remote-cache/target_compiled/" - AIRFLOW__COSMOS__REMOTE_TARGET_PATH_CONN_ID: aws_s3_conn - - - name: Upload coverage to Github - uses: actions/upload-artifact@v4 - with: - name: coverage-integration-sqlite-test-${{ matrix.python-version }}-${{ matrix.airflow-version }} - path: .coverage - include-hidden-files: true - - env: - AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ - AIRFLOW_CONN_EXAMPLE_CONN: postgres://postgres:postgres@0.0.0.0:5432/postgres - PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH +# TODO: https://github.com/astronomer/astronomer-cosmos/issues/1341 +# Run-Integration-Tests-Sqlite: +# needs: Authorize +# runs-on: ubuntu-latest +# strategy: +# matrix: +# python-version: ["3.11"] +# airflow-version: ["2.8"] +# +# steps: +# - uses: actions/checkout@v3 +# with: +# ref: ${{ github.event.pull_request.head.sha || github.ref }} +# - uses: actions/cache@v3 +# with: +# path: | +# ~/.cache/pip +# .local/share/hatch/ +# key: integration-sqlite-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.airflow-version }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('cosmos/__init__.py') }} +# +# - name: Set up Python ${{ matrix.python-version }} +# uses: actions/setup-python@v4 +# with: +# python-version: ${{ matrix.python-version }} +# +# - name: Install packages and dependencies +# run: | +# python -m pip install uv +# uv pip install --system hatch +# hatch -e tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }} run pip freeze +# +# - name: Test Cosmos against Airflow ${{ matrix.airflow-version }} and Python ${{ matrix.python-version }} +# run: | +# hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration-sqlite-setup +# hatch run tests.py${{ matrix.python-version }}-${{ matrix.airflow-version }}:test-integration-sqlite +# env: +# AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ +# AIRFLOW_CONN_EXAMPLE_CONN: postgres://postgres:postgres@0.0.0.0:5432/postgres +# AIRFLOW_CONN_AWS_S3_CONN: ${{ secrets.AIRFLOW_CONN_AWS_S3_CONN }} +# AIRFLOW_CONN_GCP_GS_CONN: ${{ secrets.AIRFLOW_CONN_GCP_GS_CONN }} +# AIRFLOW_CONN_AZURE_ABFS_CONN: ${{ secrets.AIRFLOW_CONN_AZURE_ABFS_CONN }} +# AIRFLOW__CORE__DAGBAG_IMPORT_TIMEOUT: 90.0 +# PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH +# AIRFLOW__COSMOS__ENABLE_CACHE: 0 +# COSMOS_CONN_POSTGRES_PASSWORD: ${{ secrets.COSMOS_CONN_POSTGRES_PASSWORD }} +# DATABRICKS_CLUSTER_ID: mock +# DATABRICKS_HOST: mock +# DATABRICKS_WAREHOUSE_ID: mock +# DATABRICKS_TOKEN: mock +# POSTGRES_HOST: localhost +# POSTGRES_USER: postgres +# POSTGRES_PASSWORD: postgres +# POSTGRES_DB: postgres +# POSTGRES_SCHEMA: public +# POSTGRES_PORT: 5432 +# AIRFLOW__COSMOS__REMOTE_TARGET_PATH: "s3://cosmos-remote-cache/target_compiled/" +# AIRFLOW__COSMOS__REMOTE_TARGET_PATH_CONN_ID: aws_s3_conn +# +# - name: Upload coverage to Github +# uses: actions/upload-artifact@v4 +# with: +# name: coverage-integration-sqlite-test-${{ matrix.python-version }}-${{ matrix.airflow-version }} +# path: .coverage +# include-hidden-files: true +# +# env: +# AIRFLOW_HOME: /home/runner/work/astronomer-cosmos/astronomer-cosmos/ +# AIRFLOW_CONN_EXAMPLE_CONN: postgres://postgres:postgres@0.0.0.0:5432/postgres +# PYTHONPATH: /home/runner/work/astronomer-cosmos/astronomer-cosmos/:$PYTHONPATH Run-Integration-Tests-DBT-1-5-4: needs: Authorize diff --git a/pyproject.toml b/pyproject.toml index 11290b1a5..dd73c18da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -144,6 +144,8 @@ dependencies = [ "types-requests", "types-python-dateutil", "Werkzeug<3.0.0", + "eval_type_backport", # TODO: https://github.com/astronomer/astronomer-cosmos/issues/1342 + "dbt-core<1.8.9" # TODO: https://github.com/astronomer/astronomer-cosmos/issues/1343 ] pre-install-commands = ["sh scripts/test/pre-install-airflow.sh {matrix:airflow} {matrix:python}"] diff --git a/scripts/test/integration-sqlite-setup.sh b/scripts/test/integration-sqlite-setup.sh index b8bce035c..9b0f5438d 100644 --- a/scripts/test/integration-sqlite-setup.sh +++ b/scripts/test/integration-sqlite-setup.sh @@ -1,4 +1,4 @@ pip uninstall -y dbt-core dbt-sqlite openlineage-airflow openlineage-integration-common; \ rm -rf airflow.*; \ airflow db init; \ -pip install 'dbt-core==1.4' 'dbt-sqlite<=1.4' 'dbt-databricks<=1.4' 'dbt-postgres<=1.4' +pip install 'dbt-core==1.4' 'dbt-sqlite==1.4' 'dbt-databricks==1.4' 'dbt-postgres==1.4' #'databricks-sdk==0.16.0' diff --git a/scripts/test/kubernetes-setup.sh b/scripts/test/kubernetes-setup.sh index fb2324c04..b08f37d1d 100644 --- a/scripts/test/kubernetes-setup.sh +++ b/scripts/test/kubernetes-setup.sh @@ -5,8 +5,8 @@ set -x set -e - -pip install dbt-postgres==1.8.2 psycopg2==2.9.3 pytz +# TODO: https://github.com/astronomer/astronomer-cosmos/issues/1344 +pip install 'dbt-postgres<1.8' 'psycopg2==2.9.3' 'pytz' # Create a Kubernetes secret named 'postgres-secrets' with the specified literals for host and password kubectl create secret generic postgres-secrets \ diff --git a/tests/test_example_dags.py b/tests/test_example_dags.py index e647bc2fa..642e8b165 100644 --- a/tests/test_example_dags.py +++ b/tests/test_example_dags.py @@ -79,7 +79,7 @@ def get_dag_bag() -> DagBag: file.writelines(["example_cosmos_sources.py\n"]) if DBT_VERSION < Version("1.6.0"): file.writelines(["example_model_version.py\n"]) - file.writelines(["example_clone.py\n"]) + file.writelines(["example_operators.py\n"]) if DBT_VERSION < Version("1.5.0"): file.writelines(["example_source_rendering.py\n"]) diff --git a/tests/test_example_dags_no_connections.py b/tests/test_example_dags_no_connections.py index d18d21730..59235c2d0 100644 --- a/tests/test_example_dags_no_connections.py +++ b/tests/test_example_dags_no_connections.py @@ -55,7 +55,7 @@ def get_dag_bag() -> DagBag: if DBT_VERSION < Version("1.6.0"): file.writelines(["example_model_version.py\n"]) - file.writelines(["example_clone.py\n"]) + file.writelines(["example_operators.py\n"]) # cosmos_profile_mapping uses the automatic profile rendering from an Airflow connection. # so we can't parse that without live connections for file_name in ["cosmos_profile_mapping.py"]: