Remove openlineage exclusion (apache#33491)

Now that Airflow 2.7.0 is released, we can remove exclusion that we had for openlineage which prevented from using it as dependency of Airflow in CI.
lwyszomi · Aug 18, 2023 · 008f233 · 008f233
1 parent b9c6250
commit 008f233
Show file tree

Hide file tree

Showing 10 changed files with 166 additions and 170 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -35,7 +35,7 @@
 #                        much smaller.
 #
 # Use the same builder frontend version for everyone
-ARG AIRFLOW_EXTRAS="aiobotocore,amazon,async,celery,cncf.kubernetes,daskexecutor,docker,elasticsearch,ftp,google,google_auth,grpc,hashicorp,http,ldap,microsoft.azure,mysql,odbc,pandas,postgres,redis,sendgrid,sftp,slack,snowflake,ssh,statsd,virtualenv"
+ARG AIRFLOW_EXTRAS="aiobotocore,amazon,async,celery,cncf.kubernetes,daskexecutor,docker,elasticsearch,ftp,google,google_auth,grpc,hashicorp,http,ldap,microsoft.azure,mysql,odbc,openlineage,pandas,postgres,redis,sendgrid,sftp,slack,snowflake,ssh,statsd,virtualenv"
 ARG ADDITIONAL_AIRFLOW_EXTRAS=""
 ARG ADDITIONAL_PYTHON_DEPS=""
 

diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py
@@ -360,6 +360,7 @@ def get_airflow_extras():
     "microsoft.azure",
     "mysql",
     "odbc",
+    "openlineage",
     "pandas",
     "postgres",
     "redis",

diff --git a/docs/docker-stack/build-arg-ref.rst b/docs/docker-stack/build-arg-ref.rst
@@ -98,6 +98,7 @@ List of default extras in the production Dockerfile:
 * microsoft.azure
 * mysql
 * odbc
+* openlineage
 * pandas
 * postgres
 * redis

diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt
@@ -32,10 +32,10 @@ k8s:status:1529ccd444b41c4b0b5f943289957100
 k8s:tests:2a1e2928faea2eddafaff94176a46690
 k8s:upload-k8s-image:6b3a20cdeb692f3c3d727f6b9e68c901
 k8s:c73e0ebdff75d89e35af6e324a3bc527
-prod-image:build:aba06a4d6543c41b5996de0d8738b6c7
+prod-image:build:10cc859b7d898581bf9b2a24c19c1032
 prod-image:pull:76f1f27e6119928412abecf153fce4bb
 prod-image:verify:bd2b78738a7c388dbad6076c41a9f906
-prod-image:993e54af79b73d7a9e6cfb6fc51a6e34
+prod-image:d9b47217a12f73f214bcd2938bbab84a
 release-management:add-back-references:919b3bd706acf794e0e5e129fc587f6d
 release-management:create-minor-branch:a3834afc4aa5d1e98002c9e9e7a9931d
 release-management:generate-constraints:b8fcaf8f0acd35ed5dbd48659bdb6485

diff --git a/images/breeze/output-commands.svg b/images/breeze/output-commands.svg
diff --git a/images/breeze/output_prod-image.svg b/images/breeze/output_prod-image.svg
diff --git a/images/breeze/output_prod-image_build.svg b/images/breeze/output_prod-image_build.svg
diff --git a/scripts/ci/installed_providers.txt b/scripts/ci/installed_providers.txt
@@ -14,6 +14,7 @@ imap
 microsoft.azure
 mysql
 odbc
+openlineage
 postgres
 redis
 sendgrid

diff --git a/scripts/in_container/_in_container_utils.sh b/scripts/in_container/_in_container_utils.sh
@@ -298,13 +298,6 @@ function install_all_providers_from_pypi_with_eager_upgrade() {
     for provider_package in ${ALL_PROVIDERS_PACKAGES}
     do
         echo -n "Checking if ${provider_package} is available in PyPI: "
-        if [[ ${provider_package} == "apache-airflow-providers-openlineage" ]]; then
-            # The openlineage provider has 2.7.0 airflow dependency so it should be excluded for now in
-            # "pypi" dependency calculation
-            # We should remove it right after 2.7.0 is released to PyPI and regenerate the 2.7.0 constraints
-            echo "${COLOR_YELLOW}Skipped until 2.7.0 is released${COLOR_RESET}"
-            continue
-        fi
         res=$(curl --head -s -o /dev/null -w "%{http_code}" "https://pypi.org/project/${provider_package}/")
         if [[ ${res} == "200" ]]; then
             packages_to_install+=( "${provider_package}" )

diff --git a/scripts/in_container/run_generate_constraints.sh b/scripts/in_container/run_generate_constraints.sh
@@ -93,7 +93,7 @@ elif [[ ${AIRFLOW_CONSTRAINTS_MODE} == "constraints" ]]; then
 #
 # 1. Reproducible installation of airflow with selected providers (note constraints are used):
 #
-# pip install "apache-airflow[celery,cncf.kubernetes,google,amazon,snowflake]==X.Y.Z" \
+# pip install "apache-airflow[celery,cncf.kubernetes,google,amazon,snowflake]==X.Y.Z" \\
 #     --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-X.Y.Z/constraints-3.8.txt"
 #
 # 2. Installing own dependencies that are potentially not matching the constraints (note constraints are not