From 32c47cadda80747fafeaa9b0a36d92f043f03681 Mon Sep 17 00:00:00 2001 From: Tatiana Al-Chueyr Date: Wed, 29 May 2024 09:56:20 +0100 Subject: [PATCH] Fix CI issues (#1005) Fix three problems observed while running the tests in the CI recently: 1. Static check Fix spelling issue captured by static check (#1000) 2. Integration test that's no longer needed The issue we were trying to capture for no longer happens in the latest version of the Apache Airflow provider `apache-airflow-providers-postgres==5.11.1rc1`: https://github.com/apache/airflow/issues/39842 3. Skip a buggy version of OL There was a breaking change between `openlineage-integration-common==1.14.0` and `openlineage-integration-common==1.15.0` . It may have been an unintended side-effect of https://github.com/OpenLineage/OpenLineage/pull/2693. This is an example of how Cosmos was using `DbtLocalArtifactProcessor` - something that had been agreed upon in the past: ``` openlineage_processor = DbtLocalArtifactProcessor( producer=OPENLINEAGE_PRODUCER, job_namespace=LINEAGE_NAMESPACE, project_dir=project_dir, profile_name=self.profile_config.profile_name, target=self.profile_config.target_name, ) events = openlineage_processor.parse() for completed in events.completes: for output in getattr(completed, source): dataset_uri = output.namespace + "/" + output.name uris.append(dataset_uri) ``` In `openlineage-integration-common==1.14.0` and earlier versions, this would create URIs in the format: ``` postgres://0.0.0.0:5432/postgres.public.stg_customers ``` Since openlineage-integration-common==1.15.0 , this leads to URIs being created in the format: ``` postgres.public.stg_customers/postgres://0.0.0.0:5432 ``` This was fixed in https://github.com/OpenLineage/OpenLineage/pull/2735 and released as part of OL 1.16: https://github.com/OpenLineage/OpenLineage/releases/tag/1.16.0 --- CODE_OF_CONDUCT.md | 2 +- pyproject.toml | 4 ++-- tests/operators/test_local.py | 43 ----------------------------------- 3 files changed, 3 insertions(+), 46 deletions(-) diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index aad2b3071..b3378c60a 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -5,7 +5,7 @@ We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender -identity and expression, level of experience, education, socio-economic status, +identity and expression, level of experience, education, socioeconomic status, nationality, personal appearance, race, religion, or sexual identity and orientation. diff --git a/pyproject.toml b/pyproject.toml index cb2530a5a..8cf43a98e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,7 +59,7 @@ dbt-redshift = ["dbt-redshift"] dbt-snowflake = ["dbt-snowflake"] dbt-spark = ["dbt-spark"] dbt-vertica = ["dbt-vertica<=1.5.4"] -openlineage = ["openlineage-integration-common", "openlineage-airflow"] +openlineage = ["openlineage-integration-common!=1.15.0", "openlineage-airflow"] all = ["astronomer-cosmos[dbt-all]", "astronomer-cosmos[openlineage]"] docs = [ "sphinx", @@ -121,11 +121,11 @@ packages = ["/cosmos"] [tool.hatch.envs.tests] dependencies = [ "astronomer-cosmos[tests]", - "apache-airflow-providers-postgres", "apache-airflow-providers-cncf-kubernetes>=5.1.1", "apache-airflow-providers-amazon>=3.0.0,<8.20.0", # https://github.com/apache/airflow/issues/39103 "apache-airflow-providers-docker>=3.5.0", "apache-airflow-providers-microsoft-azure", + "apache-airflow-providers-postgres", "types-PyYAML", "types-attrs", "types-requests", diff --git a/tests/operators/test_local.py b/tests/operators/test_local.py index 0f35705b6..14ed66027 100644 --- a/tests/operators/test_local.py +++ b/tests/operators/test_local.py @@ -23,7 +23,6 @@ parse_number_of_warnings_dbt_runner, parse_number_of_warnings_subprocess, ) -from cosmos.exceptions import AirflowCompatibilityError from cosmos.operators.local import ( DbtBuildLocalOperator, DbtDocsAzureStorageLocalOperator, @@ -443,48 +442,6 @@ def test_run_operator_dataset_inlets_and_outlets(caplog): assert test_operator.outlets == [] -@pytest.mark.skipif( - version.parse(airflow_version) not in PARTIALLY_SUPPORTED_AIRFLOW_VERSIONS, - reason="Airflow 2.9.0 and 2.9.1 have a breaking change in Dataset URIs", - # https://github.com/apache/airflow/issues/39486 -) -@pytest.mark.integration -def test_run_operator_dataset_emission_fails(caplog): - - with DAG("test-id-1", start_date=datetime(2022, 1, 1)) as dag: - seed_operator = DbtSeedLocalOperator( - profile_config=real_profile_config, - project_dir=DBT_PROJ_DIR, - task_id="seed", - dbt_cmd_flags=["--select", "raw_customers"], - install_deps=True, - append_env=True, - ) - run_operator = DbtRunLocalOperator( - profile_config=real_profile_config, - project_dir=DBT_PROJ_DIR, - task_id="run", - dbt_cmd_flags=["--models", "stg_customers"], - install_deps=True, - append_env=True, - ) - - seed_operator >> run_operator - - with pytest.raises(AirflowCompatibilityError) as exc: - run_test_dag(dag) - - err_msg = str(exc.value) - assert ( - "Apache Airflow 2.9.0 & 2.9.1 introduced a breaking change in Dataset URIs, to be fixed in newer versions" - in err_msg - ) - assert ( - "If you want to use Cosmos with one of these Airflow versions, you will have to disable emission of Datasets" - in err_msg - ) - - @pytest.mark.skipif( version.parse(airflow_version) not in PARTIALLY_SUPPORTED_AIRFLOW_VERSIONS, reason="Airflow 2.9.0 and 2.9.1 have a breaking change in Dataset URIs",