From bdd765d1decc0c64023f61dad0297b60d74330eb Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Tue, 25 Oct 2022 17:07:42 +0200 Subject: [PATCH] Attempt to add Python 3.11 support Python 3.11 has been released as scheduled on October 25, 2022 and this is the first attempt to see how far Airflow (mostly dependencies) are from being ready to officially support 3.11. So far we had to exclude the following dependencies: - [ ] Pyarrow dependency: https://github.com/apache/arrow/pull/14499 - [ ] Google Provider: https://github.com/apache/airflow/issues/27292 and https://github.com/googleapis/python-bigquery/issues/1386 - [ ] Databricks Provider: https://github.com/databricks/databricks-sql-python/issues/59 - [ ] Papermill Provider: https://github.com/nteract/papermill/issues/700 - [ ] Azure Provider: https://github.com/Azure/azure-uamqp-python/issues/334 and https://github.com/Azure/azure-sdk-for-python/issues/27066 - [ ] Apache Beam Provider: https://github.com/apache/beam/issues/23848 - [ ] Snowflake Provider: https://github.com/snowflakedb/snowflake-connector-python/pull/1294 - [ ] JDBC Provider: https://github.com/jpype-project/jpype/pull/1087 - [ ] Hive Provider: https://github.com/cloudera/python-sasl/issues/30 We might decide to release Airflow in 3.11 with those providers disabled in case they are lagging behind eventually, but for the moment we want to work with all the projects in concert to be able to release all providers (Google Provider requires quite a lot of work and likely Google Team stepping up and community helping with migration to latest Goofle cloud libraries) --- CI.rst | 2 +- Dockerfile | 2 +- Dockerfile.ci | 2 +- LOCAL_VIRTUALENV.rst | 12 +- README.md | 2 +- airflow/providers/apache/beam/provider.yaml | 5 + airflow/providers/apache/hive/provider.yaml | 5 + airflow/providers/databricks/provider.yaml | 6 + airflow/providers/google/provider.yaml | 6 + airflow/providers/jdbc/provider.yaml | 5 + .../providers/microsoft/azure/provider.yaml | 8 + airflow/providers/papermill/provider.yaml | 7 + airflow/providers/snowflake/provider.yaml | 7 + dev/README_RELEASE_AIRFLOW.md | 2 +- dev/breeze/SELECTIVE_CHECKS.md | 4 +- .../src/airflow_breeze/global_constants.py | 4 +- .../airflow_breeze/utils/selective_checks.py | 2 +- dev/breeze/tests/test_cache.py | 4 +- dev/breeze/tests/test_selective_checks.py | 40 +- .../prepare_provider_packages.py | 2 +- dev/retag_docker_images.py | 2 +- .../installation/prerequisites.rst | 2 +- .../upgrading-from-1-10/index.rst | 2 +- docs/docker-stack/build.rst | 16 +- generated/provider_dependencies.json | 230 +++++++---- images/breeze/output-commands-hash.txt | 68 ++-- images/breeze/output-commands.svg | 212 +++++----- images/breeze/output_ci-image.svg | 24 +- images/breeze/output_ci-image_build.svg | 292 +++++++------- images/breeze/output_ci-image_pull.svg | 152 +++---- images/breeze/output_ci-image_verify.svg | 96 ++--- images/breeze/output_ci.svg | 32 +- .../output_ci_find-newer-dependencies.svg | 104 ++--- images/breeze/output_k8s.svg | 58 +-- images/breeze/output_k8s_build-k8s-image.svg | 140 +++---- .../breeze/output_k8s_configure-cluster.svg | 140 +++---- images/breeze/output_k8s_create-cluster.svg | 156 ++++---- images/breeze/output_k8s_delete-cluster.svg | 112 +++--- images/breeze/output_k8s_deploy-airflow.svg | 176 ++++---- images/breeze/output_k8s_k9s.svg | 100 ++--- images/breeze/output_k8s_logs.svg | 112 +++--- .../breeze/output_k8s_run-complete-tests.svg | 220 +++++----- images/breeze/output_k8s_shell.svg | 116 +++--- images/breeze/output_k8s_status.svg | 116 +++--- images/breeze/output_k8s_tests.svg | 172 ++++---- images/breeze/output_k8s_upload-k8s-image.svg | 152 +++---- images/breeze/output_prod-image.svg | 24 +- images/breeze/output_prod-image_build.svg | 376 +++++++++--------- images/breeze/output_prod-image_pull.svg | 152 +++---- images/breeze/output_prod-image_verify.svg | 100 ++--- images/breeze/output_release-management.svg | 28 +- ...elease-management_generate-constraints.svg | 156 ++++---- ...release-management_release-prod-images.svg | 124 +++--- images/breeze/output_setup.svg | 26 +- images/breeze/output_setup_config.svg | 108 ++--- images/breeze/output_shell.svg | 240 +++++------ images/breeze/output_start-airflow.svg | 260 ++++++------ images/breeze/output_testing.svg | 22 +- .../output_testing_docker-compose-tests.svg | 96 ++--- images/breeze/output_testing_tests.svg | 216 +++++----- ...pre_commit_build_providers_dependencies.py | 17 +- setup.cfg | 1 + setup.py | 42 +- 63 files changed, 2667 insertions(+), 2450 deletions(-) diff --git a/CI.rst b/CI.rst index 5c937e579431f..bc790706d1106 100644 --- a/CI.rst +++ b/CI.rst @@ -59,7 +59,7 @@ Container Registry used as cache We are using GitHub Container Registry to store the results of the ``Build Images`` workflow which is used in the ``Tests`` workflow. -Currently in main version of Airflow we run tests in 4 different versions of Python (3.7, 3.8, 3.9, 3.10) +Currently in main version of Airflow we run tests in those versions of Python (3.7, 3.8, 3.9, 3.10, 3.11) which means that we have to build 8 images (4 CI ones and 4 PROD ones). Yet we run around 12 jobs with each of the CI images. That is a lot of time to just build the environment to run. Therefore we are utilising ``pull_request_target`` feature of GitHub Actions. diff --git a/Dockerfile b/Dockerfile index 9f34522ae8cc4..5368e0f831ff6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1215,7 +1215,7 @@ ARG ADDITIONAL_PYTHON_DEPS="" # are compatible with the new protobuf version. All the google python client libraries need # to be upgraded to >=2.0.0 in order to able to lift that limitation # https://developers.google.com/protocol-buffers/docs/news/2022-05-06#python-updates -ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS="dill<0.3.3 pyarrow>=6.0.0 protobuf<4.21.0" +ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS="dill<0.3.3 pyarrow>=6.0.0;python_version==\"3.10\" protobuf<4.21.0" ENV ADDITIONAL_PYTHON_DEPS=${ADDITIONAL_PYTHON_DEPS} \ INSTALL_PACKAGES_FROM_CONTEXT=${INSTALL_PACKAGES_FROM_CONTEXT} \ diff --git a/Dockerfile.ci b/Dockerfile.ci index 7446c41ac9e47..01b19892126e3 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1142,7 +1142,7 @@ RUN echo "Airflow version: ${AIRFLOW_VERSION}" # are compatible with the new protobuf version. All the google python client libraries need # to be upgraded to >= 2.0.0 in order to able to lift that limitation # https://developers.google.com/protocol-buffers/docs/news/2022-05-06#python-updates -ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS="dill<0.3.3 pyarrow>=6.0.0 protobuf<4.21.0" +ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS="dill<0.3.3 pyarrow>=6.0.0;python_version==\"3.10\" protobuf<4.21.0" ARG UPGRADE_TO_NEWER_DEPENDENCIES="false" ENV EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} \ UPGRADE_TO_NEWER_DEPENDENCIES=${UPGRADE_TO_NEWER_DEPENDENCIES} diff --git a/LOCAL_VIRTUALENV.rst b/LOCAL_VIRTUALENV.rst index 469eedbf432ff..c0f3937a13052 100644 --- a/LOCAL_VIRTUALENV.rst +++ b/LOCAL_VIRTUALENV.rst @@ -51,7 +51,7 @@ Required Software Packages Use system-level package managers like yum, apt-get for Linux, or Homebrew for macOS to install required software packages: -* Python (One of: 3.7, 3.8, 3.9, 3.10) +* Python (One of: 3.7, 3.8, 3.9, 3.10, 3.11) * MySQL 5.7+ * libxml @@ -102,7 +102,7 @@ Creating a Local virtualenv To use your IDE for Airflow development and testing, you need to configure a virtual environment. Ideally you should set up virtualenv for all Python versions that Airflow -supports (3.7, 3.8, 3.9, 3.10). +supports (3.7, 3.8, 3.9, 3.10, 3.11). To create and initialize the local virtualenv: @@ -122,7 +122,7 @@ To create and initialize the local virtualenv: .. code-block:: bash - conda create -n airflow python=3.7 # or 3.8, 3.9, 3.10 + conda create -n airflow python=3.7 # or 3.8, 3.9, 3.10, 3.11 conda activate airflow 2. Install Python PIP requirements: @@ -150,7 +150,7 @@ for different python versions). For development on current main source: .. code-block:: bash - # use the same version of python as you are working with, 3.7, 3.8, 3.9, or 3.10 + # use the same version of python as you are working with, 3.7, 3.8, 3.9, 3.10 or 3.11 pip install -e ".[devel,]" \ --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-source-providers-3.7.txt" @@ -163,7 +163,7 @@ You can also install Airflow in non-editable mode: .. code-block:: bash - # use the same version of python as you are working with, 3.7, 3.8, 3.9, or 3.10 + # use the same version of python as you are working with, 3.7, 3.8, 3.9, 3.10 or 3.11 pip install ".[devel,]" \ --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-source-providers-3.7.txt" @@ -173,7 +173,7 @@ sources, unless you set ``INSTALL_PROVIDERS_FROM_SOURCES`` environment variable .. code-block:: bash - # use the same version of python as you are working with, 3.7, 3.8, 3.9, or 3.10 + # use the same version of python as you are working with, 3.7, 3.8, 3.9, 3.10 or 3.11 INSTALL_PROVIDERS_FROM_SOURCES="true" pip install ".[devel,]" \ --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-main/constraints-source-providers-3.7.txt" diff --git a/README.md b/README.md index a9e697fca2c22..1483da292ebe6 100644 --- a/README.md +++ b/README.md @@ -87,7 +87,7 @@ Apache Airflow is tested with: | | Main version (dev) | Stable version (2.4.2) | |---------------------|------------------------------|------------------------------| -| Python | 3.7, 3.8, 3.9, 3.10 | 3.7, 3.8, 3.9, 3.10 | +| Python | 3.7, 3.8, 3.9, 3.10, 3.11 | 3.7, 3.8, 3.9, 3.10, 3.11 | | Platform | AMD64/ARM64(\*) | AMD64/ARM64(\*) | | Kubernetes | 1.21, 1.22, 1.23, 1.24, 1.25 | 1.21, 1.22, 1.23, 1.24, 1.25 | | PostgreSQL | 10, 11, 12, 13, 14 | 10, 11, 12, 13, 14 | diff --git a/airflow/providers/apache/beam/provider.yaml b/airflow/providers/apache/beam/provider.yaml index c7655bafb833b..30d23d571494f 100644 --- a/airflow/providers/apache/beam/provider.yaml +++ b/airflow/providers/apache/beam/provider.yaml @@ -38,6 +38,11 @@ dependencies: - apache-airflow>=2.3.0 - apache-beam>=2.39.0 +# Excluded because apache-beam and a number of dependent libraries are not +# compatible with 3.11 yet. See https://github.com/apache/beam/issues/23848 +excluded-python-versions: + - "3.11" + integrations: - integration-name: Apache Beam external-doc-url: https://beam.apache.org/ diff --git a/airflow/providers/apache/hive/provider.yaml b/airflow/providers/apache/hive/provider.yaml index 5c359460e02de..cd88d8c891e16 100644 --- a/airflow/providers/apache/hive/provider.yaml +++ b/airflow/providers/apache/hive/provider.yaml @@ -53,6 +53,11 @@ dependencies: - sasl>=0.3.1; python_version>="3.9" - thrift>=0.9.2 +# Excluded because SASL is not yet compatible +# with 3.11. See https://github.com/cloudera/python-sasl/issues/30 +excluded-python-versions: + - "3.11" + integrations: - integration-name: Apache Hive external-doc-url: https://hive.apache.org/ diff --git a/airflow/providers/databricks/provider.yaml b/airflow/providers/databricks/provider.yaml index 429bfc33ee72c..c09f78178c515 100644 --- a/airflow/providers/databricks/provider.yaml +++ b/airflow/providers/databricks/provider.yaml @@ -46,6 +46,12 @@ dependencies: - databricks-sql-connector>=2.0.0, <3.0.0 - aiohttp>=3.6.3, <4 +# Excluded because databricks-sql-connector and a number of dependent libraries are not compatible +# with 3.11 yet See https://pyreadiness.org/3.11/. (Pyarrow is one of the important ones) +# See https://github.com/databricks/databricks-sql-python/issues/59 for issue that tracks it +excluded-python-versions: + - "3.11" + integrations: - integration-name: Databricks external-doc-url: https://databricks.com/ diff --git a/airflow/providers/google/provider.yaml b/airflow/providers/google/provider.yaml index fb0a3d434cfeb..2f63f6ee5b8b8 100644 --- a/airflow/providers/google/provider.yaml +++ b/airflow/providers/google/provider.yaml @@ -117,6 +117,12 @@ dependencies: # Google bigtable client require protobuf <= 3.20.0. We can remove the limitation # when this limitation is removed - protobuf<=3.20.0 +# Excluded because bigquery and a number of dependent libraries are not compatible with 3.11 yet +# See https://pyreadiness.org/3.11/. Also some of Google libraries are in pretty ancient +# version (pre 2.0.0) and they MUST be migrated in order to add 3.11 support +# This is tracked in https://github.com/apache/airflow/issues/27292 +excluded-python-versions: + - "3.11" integrations: - integration-name: Google Analytics360 diff --git a/airflow/providers/jdbc/provider.yaml b/airflow/providers/jdbc/provider.yaml index 84850d248b74c..c788b39f3e303 100644 --- a/airflow/providers/jdbc/provider.yaml +++ b/airflow/providers/jdbc/provider.yaml @@ -40,6 +40,11 @@ dependencies: - apache-airflow-providers-common-sql>=1.3.0 - jaydebeapi>=1.1.1 +# Excluded because Jpype1 is not yet compatible +# with 3.11 yet. See https://github.com/jpype-project/jpype/pull/1087 +excluded-python-versions: + - "3.11" + integrations: - integration-name: Java Database Connectivity (JDBC) external-doc-url: https://docs.oracle.com/javase/8/docs/technotes/guides/jdbc/ diff --git a/airflow/providers/microsoft/azure/provider.yaml b/airflow/providers/microsoft/azure/provider.yaml index bfa336e006f11..9ae084babce17 100644 --- a/airflow/providers/microsoft/azure/provider.yaml +++ b/airflow/providers/microsoft/azure/provider.yaml @@ -65,6 +65,14 @@ dependencies: - azure-servicebus>=7.6.1; platform_machine != "aarch64" - azure-synapse-spark +# Excluded because azure-uamqp-python and azure-core and a number of dependent libraries are not compatible +# with 3.11 yet See https://pyreadiness.org/3.11/. +# See for issues that track it: +# * https://github.com/Azure/azure-uamqp-python/issues/334 +# * https://github.com/Azure/azure-sdk-for-python/issues/27066 +excluded-python-versions: + - "3.11" + integrations: - integration-name: Microsoft Azure Batch external-doc-url: https://azure.microsoft.com/en-us/services/batch/ diff --git a/airflow/providers/papermill/provider.yaml b/airflow/providers/papermill/provider.yaml index cafc3aa58783a..89645539c46f0 100644 --- a/airflow/providers/papermill/provider.yaml +++ b/airflow/providers/papermill/provider.yaml @@ -39,6 +39,13 @@ dependencies: - papermill[all]>=1.2.1 - scrapbook[all] +# Excluded because papermill and a number of dependent libraries are not compatible +# with 3.11 yet See https://pyreadiness.org/3.11/. (Pyarrow is one of the important ones) +# See for issue that tracks it https://github.com/nteract/papermill/issues/700 +excluded-python-versions: + - "3.11" + + integrations: - integration-name: Papermill external-doc-url: https://github.com/nteract/papermill diff --git a/airflow/providers/snowflake/provider.yaml b/airflow/providers/snowflake/provider.yaml index c77ee86c71f90..f07664038ad89 100644 --- a/airflow/providers/snowflake/provider.yaml +++ b/airflow/providers/snowflake/provider.yaml @@ -50,6 +50,13 @@ dependencies: - snowflake-connector-python>=2.4.1 - snowflake-sqlalchemy>=1.1.0 +# Excluded because snowflake connector python and a number of dependent libraries are not +# compatible with 3.11 yet. See https://pyreadiness.org/3.11/. One of the important dependencies +# is Pyarrow: https://github.com/apache/arrow/pull/14499 but the +# https://github.com/snowflakedb/snowflake-connector-python/issues/1289 is the issue +excluded-python-versions: + - "3.11" + integrations: - integration-name: Snowflake external-doc-url: https://snowflake.com/ diff --git a/dev/README_RELEASE_AIRFLOW.md b/dev/README_RELEASE_AIRFLOW.md index 1fd37283b738c..b3684b7b4520e 100644 --- a/dev/README_RELEASE_AIRFLOW.md +++ b/dev/README_RELEASE_AIRFLOW.md @@ -986,7 +986,7 @@ the older branches, you should set the "skip" field to true. ## Verify production images ```shell script -for PYTHON in 3.7 3.8 3.9 3.10 +for PYTHON in 3.7 3.8 3.9 3.10 3.11 do docker pull apache/airflow:${VERSION}-python${PYTHON} breeze prod-image verify --image-name apache/airflow:${VERSION}-python${PYTHON} diff --git a/dev/breeze/SELECTIVE_CHECKS.md b/dev/breeze/SELECTIVE_CHECKS.md index fbdbb46806a8c..797e2ec93c047 100644 --- a/dev/breeze/SELECTIVE_CHECKS.md +++ b/dev/breeze/SELECTIVE_CHECKS.md @@ -109,8 +109,8 @@ The selective check outputs available are described below: | Output | Meaning of the output | Example value | |------------------------------------|--------------------------------------------------------------------------------------------------------|---------------------------------------------------------------| -| all-python-versions | List of all python versions there are available in the form of JSON array | ['3.7', '3.8', '3.9', '3.10'] | -| all-python-versions-list-as-string | List of all python versions there are available in the form of space separated string | 3.7 3.8 3.9 3.10 | +| all-python-versions | List of all python versions there are available in the form of JSON array | ['3.7', '3.8', '3.9', '3.10', '3.11'] | +| all-python-versions-list-as-string | List of all python versions there are available in the form of space separated string | 3.7 3.8 3.9 3.10 3.11 | | basic-checks-only | Whether to run all static checks ("false") or only basic set of static checks ("true") | false | | cache-directive | Which cache should be be used for images ("registry", "local" , "disabled") | registry | | debug-resources | Whether resources usage should be printed during parallel job execution ("true"/ "false") | false | diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index 27e0d5e318ab7..72e46c605ba6a 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -37,7 +37,7 @@ APACHE_AIRFLOW_GITHUB_REPOSITORY = "apache/airflow" # Checked before putting in build cache -ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS = ["3.7", "3.8", "3.9", "3.10"] +ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11"] DEFAULT_PYTHON_MAJOR_MINOR_VERSION = ALLOWED_PYTHON_MAJOR_MINOR_VERSIONS[0] ALLOWED_ARCHITECTURES = [Architecture.X86_64, Architecture.ARM] ALLOWED_BACKENDS = ["sqlite", "mysql", "postgres", "mssql"] @@ -153,7 +153,7 @@ def get_default_platform_machine() -> str: PYTHONDONTWRITEBYTECODE = True PRODUCTION_IMAGE = False -ALL_PYTHON_MAJOR_MINOR_VERSIONS = ["3.7", "3.8", "3.9", "3.10"] +ALL_PYTHON_MAJOR_MINOR_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11"] CURRENT_PYTHON_MAJOR_MINOR_VERSIONS = ALL_PYTHON_MAJOR_MINOR_VERSIONS CURRENT_POSTGRES_VERSIONS = ["10", "11", "12", "13", "14"] DEFAULT_POSTGRES_VERSION = CURRENT_POSTGRES_VERSIONS[0] diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index d1bc8d4dc2beb..91bae71391b66 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -360,7 +360,7 @@ def mysql_exclude(self) -> list[dict[str, str]]: @cached_property def sqlite_exclude(self) -> list[dict[str, str]]: - return [{"python-version": "3.9"}] if self._full_tests_needed else [] + return [{"python-version": "3.11"}] if self._full_tests_needed else [] @cached_property def kubernetes_versions(self) -> list[str]: diff --git a/dev/breeze/tests/test_cache.py b/dev/breeze/tests/test_cache.py index a37a91165b4fc..1db27494abd07 100644 --- a/dev/breeze/tests/test_cache.py +++ b/dev/breeze/tests/test_cache.py @@ -36,8 +36,8 @@ [ ("backend", "mysql", (True, ["sqlite", "mysql", "postgres", "mssql"]), None), ("backend", "xxx", (False, ["sqlite", "mysql", "postgres", "mssql"]), None), - ("python_major_minor_version", "3.8", (True, ["3.7", "3.8", "3.9", "3.10"]), None), - ("python_major_minor_version", "3.5", (False, ["3.7", "3.8", "3.9", "3.10"]), None), + ("python_major_minor_version", "3.8", (True, ["3.7", "3.8", "3.9", "3.10", "3.11"]), None), + ("python_major_minor_version", "3.5", (False, ["3.7", "3.8", "3.9", "3.10", "3.11"]), None), ("missing", "value", None, AttributeError), ], ) diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 8ae602be2e318..5c74e9b504b9f 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -206,8 +206,8 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): pytest.param( ("setup.py",), { - "all-python-versions": "['3.7', '3.8', '3.9', '3.10']", - "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10", + "all-python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']", + "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11", "image-build": "true", "needs-helm-tests": "true", "run-tests": "true", @@ -223,8 +223,8 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): pytest.param( ("generated/provider_dependencies.json",), { - "all-python-versions": "['3.7', '3.8', '3.9', '3.10']", - "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10", + "all-python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']", + "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11", "image-build": "true", "needs-helm-tests": "true", "run-tests": "true", @@ -260,8 +260,8 @@ def test_expected_output_pull_request_main( ("full tests needed",), "main", { - "all-python-versions": "['3.7', '3.8', '3.9', '3.10']", - "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10", + "all-python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']", + "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11", "image-build": "true", "run-tests": "true", "docs-build": "true", @@ -280,8 +280,8 @@ def test_expected_output_pull_request_main( ), "main", { - "all-python-versions": "['3.7', '3.8', '3.9', '3.10']", - "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10", + "all-python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']", + "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11", "image-build": "true", "run-tests": "true", "docs-build": "true", @@ -298,8 +298,8 @@ def test_expected_output_pull_request_main( ("full tests needed",), "main", { - "all-python-versions": "['3.7', '3.8', '3.9', '3.10']", - "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10", + "all-python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']", + "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11", "image-build": "true", "run-tests": "true", "docs-build": "true", @@ -316,8 +316,8 @@ def test_expected_output_pull_request_main( ("full tests needed",), "v2-3-stable", { - "all-python-versions": "['3.7', '3.8', '3.9', '3.10']", - "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10", + "all-python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']", + "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11", "image-build": "true", "run-tests": "true", "docs-build": "true", @@ -513,8 +513,8 @@ def test_expected_output_pull_request_target( (), "main", { - "all-python-versions": "['3.7', '3.8', '3.9', '3.10']", - "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10", + "all-python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']", + "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11", "image-build": "true", "needs-helm-tests": "true", "run-tests": "true", @@ -529,8 +529,8 @@ def test_expected_output_pull_request_target( (), "v2-3-stable", { - "all-python-versions": "['3.7', '3.8', '3.9', '3.10']", - "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10", + "all-python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']", + "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11", "image-build": "true", "needs-helm-tests": "false", "run-tests": "true", @@ -546,8 +546,8 @@ def test_expected_output_pull_request_target( (), "main", { - "all-python-versions": "['3.7', '3.8', '3.9', '3.10']", - "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10", + "all-python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']", + "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11", "image-build": "true", "needs-helm-tests": "true", "run-tests": "true", @@ -595,8 +595,8 @@ def test_no_commit_provided_trigger_full_build_for_any_event_type(github_event): ) assert_outputs_are_printed( { - "all-python-versions": "['3.7', '3.8', '3.9', '3.10']", - "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10", + "all-python-versions": "['3.7', '3.8', '3.9', '3.10', '3.11']", + "all-python-versions-list-as-string": "3.7 3.8 3.9 3.10 3.11", "image-build": "true", "needs-helm-tests": "true", "run-tests": "true", diff --git a/dev/provider_packages/prepare_provider_packages.py b/dev/provider_packages/prepare_provider_packages.py index 43d32e99c7abc..b10961f6f97d2 100755 --- a/dev/provider_packages/prepare_provider_packages.py +++ b/dev/provider_packages/prepare_provider_packages.py @@ -53,7 +53,7 @@ from rich.syntax import Syntax from yaml import safe_load -ALL_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10"] +ALL_PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11"] INITIAL_CHANGELOG_CONTENT = """ diff --git a/dev/retag_docker_images.py b/dev/retag_docker_images.py index db203fd741972..4bc72ae28a2e4 100755 --- a/dev/retag_docker_images.py +++ b/dev/retag_docker_images.py @@ -31,7 +31,7 @@ import rich_click as click -PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10"] +PYTHON_VERSIONS = ["3.7", "3.8", "3.9", "3.10", "3.11"] GHCR_IO_PREFIX = "ghcr.io" diff --git a/docs/apache-airflow/installation/prerequisites.rst b/docs/apache-airflow/installation/prerequisites.rst index f4bf65d208ae4..af1a92d9cba46 100644 --- a/docs/apache-airflow/installation/prerequisites.rst +++ b/docs/apache-airflow/installation/prerequisites.rst @@ -20,7 +20,7 @@ Prerequisites Starting with Airflow 2.3.0, Airflow is tested with:. -* Python: 3.7, 3.8, 3.9, 3.10 +* Python: 3.7, 3.8, 3.9, 3.10, 3.11 * Databases: diff --git a/docs/apache-airflow/upgrading-from-1-10/index.rst b/docs/apache-airflow/upgrading-from-1-10/index.rst index 081a902d1903f..e49e925bab948 100644 --- a/docs/apache-airflow/upgrading-from-1-10/index.rst +++ b/docs/apache-airflow/upgrading-from-1-10/index.rst @@ -34,7 +34,7 @@ Airflow 1.10 was the last release series to support Python 2. Airflow 2.0.0 requires Python 3.6+ and has been tested with Python versions 3.6, 3.7 and 3.8. Python 3.9 support was added from Airflow 2.1.2. -Airflow 2.3.0 dropped support for Python 3.6. It's tested with Python 3.7, 3.8, 3.9 and 3.10. +Airflow 2.3.0 dropped support for Python 3.6. It's tested with Python 3.7, 3.8, 3.9, 3.10, 3.11. If you have a specific task that still requires Python 2 then you can use the ``@task.virtualenv``, ``@task.docker`` or ``@task.kubernetes`` decorators for this. diff --git a/docs/docker-stack/build.rst b/docs/docker-stack/build.rst index 5849e17709846..38e8ff8b898de 100644 --- a/docs/docker-stack/build.rst +++ b/docs/docker-stack/build.rst @@ -262,14 +262,14 @@ There are two types of images you can extend your image from: Naming conventions for the images: -+----------------+------------------+---------------------------------+--------------------------------------+ -| Image | Python | Standard image | Slim image | -+================+==================+=================================+======================================+ -| Latest default | 3.7 | apache/airflow:latest | apache/airflow:slim-latest | -| Default | 3.7 | apache/airflow:X.Y.Z | apache/airflow:slim-X.Y.Z | -| Latest | 3.7,3.8,3.9,3.10 | apache/airflow:latest-pythonN.M | apache/airflow:slim-latest-pythonN.M | -| Specific | 3.7,3.8,3.9,3.10 | apache/airflow:X.Y.Z-pythonN.M | apache/airflow:slim-X.Y.Z-pythonN.M | -+----------------+------------------+---------------------------------+--------------------------------------+ ++----------------+-----------------------+---------------------------------+--------------------------------------+ +| Image | Python | Standard image | Slim image | ++================+=======================+=================================+======================================+ +| Latest default | 3.7 | apache/airflow:latest | apache/airflow:slim-latest | +| Default | 3.7 | apache/airflow:X.Y.Z | apache/airflow:slim-X.Y.Z | +| Latest | 3.7,3.8,3.9,3.10,3.11 | apache/airflow:latest-pythonN.M | apache/airflow:slim-latest-pythonN.M | +| Specific | 3.7,3.8,3.9,3.10,3.11 | apache/airflow:X.Y.Z-pythonN.M | apache/airflow:slim-X.Y.Z-pythonN.M | ++----------------+-----------------------+---------------------------------+--------------------------------------+ * The "latest" image is always the latest released stable version available. diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index c3238f556ea9b..3c7e142dfa028 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -6,14 +6,16 @@ ], "cross-providers-deps": [ "http" - ] + ], + "excluded-python-versions": [] }, "alibaba": { "deps": [ "apache-airflow>=2.3.0", "oss2>=2.14.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "amazon": { "deps": [ @@ -40,7 +42,8 @@ "mongo", "salesforce", "ssh" - ] + ], + "excluded-python-versions": [] }, "apache.beam": { "deps": [ @@ -49,6 +52,9 @@ ], "cross-providers-deps": [ "google" + ], + "excluded-python-versions": [ + "3.11" ] }, "apache.cassandra": { @@ -56,7 +62,8 @@ "apache-airflow>=2.3.0", "cassandra-driver>=3.13.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "apache.drill": { "deps": [ @@ -66,7 +73,8 @@ ], "cross-providers-deps": [ "common.sql" - ] + ], + "excluded-python-versions": [] }, "apache.druid": { "deps": [ @@ -77,7 +85,8 @@ "cross-providers-deps": [ "apache.hive", "common.sql" - ] + ], + "excluded-python-versions": [] }, "apache.hdfs": { "deps": [ @@ -85,7 +94,8 @@ "hdfs[avro,dataframe,kerberos]>=2.0.4", "snakebite-py3" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "apache.hive": { "deps": [ @@ -105,6 +115,9 @@ "presto", "samba", "vertica" + ], + "excluded-python-versions": [ + "3.11" ] }, "apache.kylin": { @@ -112,7 +125,8 @@ "apache-airflow>=2.3.0", "kylinpy>=2.6" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "apache.livy": { "deps": [ @@ -121,13 +135,15 @@ ], "cross-providers-deps": [ "http" - ] + ], + "excluded-python-versions": [] }, "apache.pig": { "deps": [ "apache-airflow>=2.3.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "apache.pinot": { "deps": [ @@ -137,41 +153,47 @@ ], "cross-providers-deps": [ "common.sql" - ] + ], + "excluded-python-versions": [] }, "apache.spark": { "deps": [ "apache-airflow>=2.3.0", "pyspark" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "apache.sqoop": { "deps": [ "apache-airflow>=2.3.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "arangodb": { "deps": [ "apache-airflow>=2.3.0", "python-arango>=7.3.2" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "asana": { "deps": [ "apache-airflow>=2.3.0", "asana>=0.10" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "atlassian.jira": { "deps": [ "JIRA>1.0.7", "apache-airflow>=2.3.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "celery": { "deps": [ @@ -179,14 +201,16 @@ "celery>=5.2.3,<6", "flower>=1.0.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "cloudant": { "deps": [ "apache-airflow>=2.3.0", "cloudant>=2.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "cncf.kubernetes": { "deps": [ @@ -194,13 +218,15 @@ "cryptography>=2.0.0", "kubernetes>=21.7.0,<24" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "common.sql": { "deps": [ "sqlparse>=0.4.2" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "databricks": { "deps": [ @@ -212,6 +238,9 @@ ], "cross-providers-deps": [ "common.sql" + ], + "excluded-python-versions": [ + "3.11" ] }, "datadog": { @@ -219,7 +248,8 @@ "apache-airflow>=2.3.0", "datadog>=0.14.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "dbt.cloud": { "deps": [ @@ -228,7 +258,8 @@ ], "cross-providers-deps": [ "http" - ] + ], + "excluded-python-versions": [] }, "dingding": { "deps": [ @@ -237,7 +268,8 @@ ], "cross-providers-deps": [ "http" - ] + ], + "excluded-python-versions": [] }, "discord": { "deps": [ @@ -246,14 +278,16 @@ ], "cross-providers-deps": [ "http" - ] + ], + "excluded-python-versions": [] }, "docker": { "deps": [ "apache-airflow>=2.4.0", "docker>=5.0.3" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "elasticsearch": { "deps": [ @@ -265,7 +299,8 @@ ], "cross-providers-deps": [ "common.sql" - ] + ], + "excluded-python-versions": [] }, "exasol": { "deps": [ @@ -276,25 +311,29 @@ ], "cross-providers-deps": [ "common.sql" - ] + ], + "excluded-python-versions": [] }, "facebook": { "deps": [ "apache-airflow>=2.3.0", "facebook-business>=6.0.2" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "ftp": { "deps": [], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "github": { "deps": [ "apache-airflow>=2.3.0", "pygithub" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "google": { "deps": [ @@ -367,6 +406,9 @@ "sftp", "ssh", "trino" + ], + "excluded-python-versions": [ + "3.11" ] }, "grpc": { @@ -376,7 +418,8 @@ "google-auth>=1.0.0, <3.0.0", "grpcio>=1.15.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "hashicorp": { "deps": [ @@ -385,18 +428,21 @@ ], "cross-providers-deps": [ "google" - ] + ], + "excluded-python-versions": [] }, "http": { "deps": [ "requests>=2.26.0", "requests_toolbelt" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "imap": { "deps": [], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "influxdb": { "deps": [ @@ -404,7 +450,8 @@ "influxdb-client>=1.19.0", "requests>=2.26.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "jdbc": { "deps": [ @@ -414,6 +461,9 @@ ], "cross-providers-deps": [ "common.sql" + ], + "excluded-python-versions": [ + "3.11" ] }, "jenkins": { @@ -421,7 +471,8 @@ "apache-airflow>=2.3.0", "python-jenkins>=1.0.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "microsoft.azure": { "deps": [ @@ -446,6 +497,9 @@ "google", "oracle", "sftp" + ], + "excluded-python-versions": [ + "3.11" ] }, "microsoft.mssql": { @@ -456,20 +510,23 @@ ], "cross-providers-deps": [ "common.sql" - ] + ], + "excluded-python-versions": [] }, "microsoft.psrp": { "deps": [ "pypsrp>=0.8.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "microsoft.winrm": { "deps": [ "apache-airflow>=2.3.0", "pywinrm>=0.4" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "mongo": { "deps": [ @@ -477,7 +534,8 @@ "dnspython>=1.13.0", "pymongo>=3.6.0,<4.0.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "mysql": { "deps": [ @@ -492,14 +550,16 @@ "presto", "trino", "vertica" - ] + ], + "excluded-python-versions": [] }, "neo4j": { "deps": [ "apache-airflow>=2.3.0", "neo4j>=4.2.1" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "odbc": { "deps": [ @@ -509,20 +569,23 @@ ], "cross-providers-deps": [ "common.sql" - ] + ], + "excluded-python-versions": [] }, "openfaas": { "deps": [ "apache-airflow>=2.3.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "opsgenie": { "deps": [ "apache-airflow>=2.3.0", "opsgenie-sdk>=2.1.5" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "oracle": { "deps": [ @@ -532,14 +595,16 @@ ], "cross-providers-deps": [ "common.sql" - ] + ], + "excluded-python-versions": [] }, "pagerduty": { "deps": [ "apache-airflow>=2.3.0", "pdpyras>=4.1.2" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "papermill": { "deps": [ @@ -547,14 +612,18 @@ "papermill[all]>=1.2.1", "scrapbook[all]" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [ + "3.11" + ] }, "plexus": { "deps": [ "apache-airflow>=2.3.0", "arrow>=0.16.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "postgres": { "deps": [ @@ -565,7 +634,8 @@ "cross-providers-deps": [ "amazon", "common.sql" - ] + ], + "excluded-python-versions": [] }, "presto": { "deps": [ @@ -577,7 +647,8 @@ "cross-providers-deps": [ "common.sql", "google" - ] + ], + "excluded-python-versions": [] }, "qubole": { "deps": [ @@ -587,14 +658,16 @@ ], "cross-providers-deps": [ "common.sql" - ] + ], + "excluded-python-versions": [] }, "redis": { "deps": [ "apache-airflow>=2.3.0", "redis~=3.2" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "salesforce": { "deps": [ @@ -602,28 +675,32 @@ "pandas>=0.17.1", "simple-salesforce>=1.0.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "samba": { "deps": [ "apache-airflow>=2.3.0", "smbprotocol>=1.5.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "segment": { "deps": [ "analytics-python>=1.2.9", "apache-airflow>=2.3.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "sendgrid": { "deps": [ "apache-airflow>=2.3.0", "sendgrid>=6.0.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "sftp": { "deps": [ @@ -632,14 +709,16 @@ ], "cross-providers-deps": [ "ssh" - ] + ], + "excluded-python-versions": [] }, "singularity": { "deps": [ "apache-airflow>=2.3.0", "spython>=0.0.56" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "slack": { "deps": [ @@ -649,7 +728,8 @@ ], "cross-providers-deps": [ "common.sql" - ] + ], + "excluded-python-versions": [] }, "snowflake": { "deps": [ @@ -661,6 +741,9 @@ "cross-providers-deps": [ "common.sql", "slack" + ], + "excluded-python-versions": [ + "3.11" ] }, "sqlite": { @@ -669,7 +752,8 @@ ], "cross-providers-deps": [ "common.sql" - ] + ], + "excluded-python-versions": [] }, "ssh": { "deps": [ @@ -677,27 +761,31 @@ "paramiko>=2.6.0", "sshtunnel>=0.3.2" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "tableau": { "deps": [ "apache-airflow>=2.3.0", "tableauserverclient" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "tabular": { "deps": [ "apache-airflow>=2.3.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "telegram": { "deps": [ "apache-airflow>=2.3.0", "python-telegram-bot>=13.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "trino": { "deps": [ @@ -709,7 +797,8 @@ "cross-providers-deps": [ "common.sql", "google" - ] + ], + "excluded-python-versions": [] }, "vertica": { "deps": [ @@ -719,20 +808,23 @@ ], "cross-providers-deps": [ "common.sql" - ] + ], + "excluded-python-versions": [] }, "yandex": { "deps": [ "apache-airflow>=2.3.0", "yandexcloud>=0.173.0" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] }, "zendesk": { "deps": [ "apache-airflow>=2.3.0", "zenpy>=2.0.24" ], - "cross-providers-deps": [] + "cross-providers-deps": [], + "excluded-python-versions": [] } } diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index b80e0c1dfa253..dc671da042b5e 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -1,58 +1,58 @@ # This file is automatically generated by pre-commit. If you have a conflict with this file # Please do not solve it but run `breeze setup regenerate-command-images`. # This command should fix the conflict and regenerate help images that you have conflict with. -main:7322af669ed8bab9cc53ff82372d7b6d +main:5cfbfcac22b77a39c00831c15945e4d6 build-docs:247bea182253970e95dd69d3ccdcbf71 -ci:find-newer-dependencies:8fa2b57f5f0523c928743b235ee3ab5a +ci:find-newer-dependencies:0c29a13f4f2bbccbd72d946f94eaa20a ci:fix-ownership:84902165a54467564fbdd3598fa273e2 ci:free-space:bb8e7ac63d12ab3ede272a898de2f527 ci:get-workflow-info:01ee34c33ad62fa5dc33e0ac8773223f ci:resource-check:0fb929ac3496dbbe97acfe99e35accd7 ci:selective-check:d4e3c250cd6f2b0040fbe6557fa423f6 -ci:6d3ac64938416e047a435ce42db6cd06 -ci-image:build:b426917e238a6c7532c2eac1108b8512 -ci-image:pull:73266206a4d1908f389352366bacec7f -ci-image:verify:e582cf1b2c00716442b6409ca8641264 -ci-image:9c271032f68105c201b7eda80f1a867e +ci:de4c7ccbf8185c470db3ab60f0de72f5 +ci-image:build:611c6eb59444b25d49ff372101525e7a +ci-image:pull:490cf296666f268a992e9ca598b7a3df +ci-image:verify:8749454475b69eaedc45a61d1fc7e681 +ci-image:0922c573ce825c2417af901238f0ff5e cleanup:9bf46a1dfd9db4fe13a1c233ad1bb96b compile-www-assets:23675c1862d0968cbff6ab6f1d93d488 exec:89b81bc34d45b0fe6653a6db5482258c -k8s:build-k8s-image:490592aa0af35b4a035f5fcbd1d1eba4 -k8s:configure-cluster:565765c4fd0c3c6da4db066e7bf34dc5 -k8s:create-cluster:575ed2d649c35fde27fd7fd39e4ae4d6 -k8s:delete-cluster:3c1165914eeec4729c4e2f325853edfa -k8s:deploy-airflow:97db729690c3d9e88953fce12602fb2b -k8s:k9s:64be156ce5cea8f1286c074e590f57e7 -k8s:logs:6e666b2318fd809724dc1f6a23661cb8 -k8s:run-complete-tests:30c2041780fff1d34e5087a7daa91eab +k8s:build-k8s-image:2f44ff20b9300afb0d8a71067b7e2cd6 +k8s:configure-cluster:f95cb1240a2d4a63b63f01a213462a43 +k8s:create-cluster:c828ef9c6e2e1ee0ad1cb3cc2ad1e637 +k8s:delete-cluster:f2470ce1458d6e10eaf6f0fa9878a759 +k8s:deploy-airflow:16c92cf48194bca09e60cc9d2fc6d70a +k8s:k9s:372b4142fba72465faabf9145de9f610 +k8s:logs:35909a3c098a4ebabadf3dee0abf4c0e +k8s:run-complete-tests:ac072d7bb9dfb0a154a534d64abd4318 k8s:setup-env:81077d4a377af3f5525d134c72be4707 -k8s:shell:3e06c69cf16b174888fcc5d76b34dbef -k8s:status:08c5dba9d6bd155ab6f537b1054d7f2b -k8s:tests:55b5353aa92691f9be9087bdf0f5d60d -k8s:upload-k8s-image:6a7fe33c173f3a60579a689d299971eb -k8s:539209e41e2d4e15bcc02f60e3b51568 -prod-image:build:892708a9dd365e7e091b7a8daea87bd4 -prod-image:pull:ba0f383e5adbb1a8a295138ec7f88038 -prod-image:verify:4d40bd67bc559d6cab253dee34ce1856 -prod-image:8d0a9dce9dab4c936f061a622152fc99 -release-management:generate-constraints:d4acf4ba8a33fbf9579265aa2ac8f788 +k8s:shell:4d3384b367a6f42cc006db1f1a00363b +k8s:status:5e8bb5648d11184014e50b8a6d9c867f +k8s:tests:582011a96a41d6ed3d8b55be720f7e2f +k8s:upload-k8s-image:21bd83464db4ebc8b8c3e747746e84b9 +k8s:d65b5534dd22fbe27b304070b59d1c4a +prod-image:build:97e31f09a8b3cf62e99400697ac0a8db +prod-image:pull:3abea52c86bc46bae112c04e45bf15a9 +prod-image:verify:3362d21e2a04d7acb2867f437f602d54 +prod-image:7c7d6e3c5ecd1eb94508d1620eaf9601 +release-management:generate-constraints:23b2c7e92237908271d5787ee838ead7 release-management:prepare-airflow-package:cff9d88ca313db10f3cc464c6798f6be release-management:prepare-provider-documentation:bc6da01ccff04fdee4f0f0de10944a4c release-management:prepare-provider-packages:ef7084750991a72a1594537c6bce8efb -release-management:release-prod-images:8858fe5a13989c7c65a79dc97a880928 +release-management:release-prod-images:fee8568f6f2aefd82dab2a675dc26759 release-management:verify-provider-packages:797e60067fc4611112527de808b5c1c1 -release-management:e6225d878ab2459d9e832702e5a37fb4 +release-management:82ca7a6ec93a12f5f40f368a87cf58a3 setup:autocomplete:9787fd9ac963e3c29e54ac53ec2b9565 -setup:config:92653afc11889e1b78e3a2e38f41107f +setup:config:f1c842220550d2b16032a4dd0c6894ae setup:regenerate-command-images:e4eb184bbc0c54683eaf38972c31870d setup:self-upgrade:d02f70c7a230eae3463ceec2056b63fa setup:version:d11da4c17a23179830079b646160149c -setup:a6bccf7a73bfac49ce5b027c8900ea61 -shell:557ee58e7c70af052f5ea7a173b39137 -start-airflow:b089191910e9c3f2ffd9c0079cdf38c6 +setup:1ec602b7b53e788687cdc85cd2d28eb2 +shell:73bdc84039673d3e0a0d442e68080166 +start-airflow:83742bfbd2e71b0fd8f803e046a3f595 static-checks:8acd42315c78e91537634a54095ee3e6 stop:8ebd8a42f1003495d37b884de5ac7ce6 -testing:docker-compose-tests:70c744105ff61025f25d93a2f1f427c2 +testing:docker-compose-tests:370a90151e2fd63d3a27ae70ae350c71 testing:helm-tests:35f7ecef86fd9c9dbad73f20ebd64496 -testing:tests:dbca53292ca6b3fbd068a62d6478d6d7 -testing:7abaa71b4c100242212ef2c0005c1a30 +testing:tests:0324bdf7a770e7bc1d1398756d93e294 +testing:223a9679ba9d56b458229143aa3f5a92 diff --git a/images/breeze/output-commands.svg b/images/breeze/output-commands.svg index 9986e417f32e9..3f29a134e900c 100644 --- a/images/breeze/output-commands.svg +++ b/images/breeze/output-commands.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - Breeze commands + Breeze commands - + - - -Usage: breeze [OPTIONS] COMMAND [ARGS]... - -╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ -│--postgres-version-PVersion of Postgres used.(>10< | 11 | 12 | 13 | 14)[default: 10]│ -│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ -│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ -│--integrationIntegration(s) to enable when running (can be more than one).                            │ -│(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino |   │ -│all)                                                                                     │ -│--forward-credentials-fForward local credentials to container when running.│ -│--db-reset-dReset DB when entering the container.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--max-timeMaximum time that the command should take - if it takes longer, the command will fail.│ -│(INTEGER RANGE)                                                                       │ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Basic developer commands ───────────────────────────────────────────────────────────────────────────────────────────╮ -│start-airflow     Enter breeze environment and starts all Airflow components in the tmux session. Compile assets  │ -│if contents of www directory changed.                                                           │ -│static-checks     Run static checks.                                                                              │ -│build-docs        Build documentation in the container.                                                           │ -│stop              Stop running breeze environment.                                                                │ -│shell             Enter breeze environment. this is the default command use when no other is selected.            │ -│exec              Joins the interactive shell of running airflow container.                                       │ -│compile-www-assetsCompiles www assets.                                                                            │ -│cleanup           Cleans the cache of parameters, docker cache and optionally built CI/PROD images.               │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced command groups ────────────────────────────────────────────────────────────────────────────────────────────╮ -│testing                Tools that developers can use to run tests                                                 │ -│ci-image               Tools that developers can use to manually manage CI images                                 │ -│k8s                    Tools that developers use to run Kubernetes tests                                          │ -│prod-image             Tools that developers can use to manually manage PROD images                               │ -│setup                  Tools that developers can use to configure Breeze                                          │ -│release-management     Tools that release managers can use to prepare and manage Airflow releases                 │ -│ci                     Tools that CI workflows use to cleanup/manage CI environment                               │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze [OPTIONS] COMMAND [ARGS]... + +╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.7< | 3.8 | 3.9 | 3.10 | 3.11)                           │ +│[default: 3.7]                                              │ +│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ +│--postgres-version-PVersion of Postgres used.(>10< | 11 | 12 | 13 | 14)[default: 10]│ +│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ +│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ +│--integrationIntegration(s) to enable when running (can be more than one).                            │ +│(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino |   │ +│all)                                                                                     │ +│--forward-credentials-fForward local credentials to container when running.│ +│--db-reset-dReset DB when entering the container.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--max-timeMaximum time that the command should take - if it takes longer, the command will fail.│ +│(INTEGER RANGE)                                                                       │ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Basic developer commands ───────────────────────────────────────────────────────────────────────────────────────────╮ +│start-airflow     Enter breeze environment and starts all Airflow components in the tmux session. Compile assets  │ +│if contents of www directory changed.                                                           │ +│static-checks     Run static checks.                                                                              │ +│build-docs        Build documentation in the container.                                                           │ +│stop              Stop running breeze environment.                                                                │ +│shell             Enter breeze environment. this is the default command use when no other is selected.            │ +│exec              Joins the interactive shell of running airflow container.                                       │ +│compile-www-assetsCompiles www assets.                                                                            │ +│cleanup           Cleans the cache of parameters, docker cache and optionally built CI/PROD images.               │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced command groups ────────────────────────────────────────────────────────────────────────────────────────────╮ +│testing                Tools that developers can use to run tests                                                 │ +│ci-image               Tools that developers can use to manually manage CI images                                 │ +│k8s                    Tools that developers use to run Kubernetes tests                                          │ +│prod-image             Tools that developers can use to manually manage PROD images                               │ +│setup                  Tools that developers can use to configure Breeze                                          │ +│release-management     Tools that release managers can use to prepare and manage Airflow releases                 │ +│ci                     Tools that CI workflows use to cleanup/manage CI environment                               │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_ci-image.svg b/images/breeze/output_ci-image.svg index db16ef837f146..ab881d6444eb0 100644 --- a/images/breeze/output_ci-image.svg +++ b/images/breeze/output_ci-image.svg @@ -35,8 +35,8 @@ .terminal-3949946042-r1 { fill: #c5c8c6;font-weight: bold } .terminal-3949946042-r2 { fill: #c5c8c6 } .terminal-3949946042-r3 { fill: #d0b344;font-weight: bold } -.terminal-3949946042-r4 { fill: #868887 } -.terminal-3949946042-r5 { fill: #68a0b3;font-weight: bold } +.terminal-3949946042-r4 { fill: #68a0b3;font-weight: bold } +.terminal-3949946042-r5 { fill: #868887 } .terminal-3949946042-r6 { fill: #98a84b;font-weight: bold } @@ -93,18 +93,18 @@ -Usage: breeze ci-image [OPTIONS] COMMAND [ARGS]... +Usage: breeze ci-image [OPTIONS] COMMAND [ARGS]... -Tools that developers can use to manually manage CI images +Tools that developers can use to manually manage CI images -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ CI Image tools ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│build   Build CI image. Include building multiple images for all python versions.                                 │ -│pull    Pull and optionally verify CI images - possibly in parallel for all Python versions.                      │ -│verify  Verify CI image.                                                                                          │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ CI Image tools ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│build   Build CI image. Include building multiple images for all python versions.                                 │ +│pull    Pull and optionally verify CI images - possibly in parallel for all Python versions.                      │ +│verify  Verify CI image.                                                                                          │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_ci-image_build.svg b/images/breeze/output_ci-image_build.svg index e3b548c176df2..035ad996c6562 100644 --- a/images/breeze/output_ci-image_build.svg +++ b/images/breeze/output_ci-image_build.svg @@ -19,309 +19,309 @@ font-weight: 700; } - .terminal-1761125991-matrix { + .terminal-3503597385-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1761125991-title { + .terminal-3503597385-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1761125991-r1 { fill: #c5c8c6;font-weight: bold } -.terminal-1761125991-r2 { fill: #c5c8c6 } -.terminal-1761125991-r3 { fill: #d0b344;font-weight: bold } -.terminal-1761125991-r4 { fill: #868887 } -.terminal-1761125991-r5 { fill: #68a0b3;font-weight: bold } -.terminal-1761125991-r6 { fill: #98a84b;font-weight: bold } -.terminal-1761125991-r7 { fill: #8d7b39 } + .terminal-3503597385-r1 { fill: #c5c8c6;font-weight: bold } +.terminal-3503597385-r2 { fill: #c5c8c6 } +.terminal-3503597385-r3 { fill: #d0b344;font-weight: bold } +.terminal-3503597385-r4 { fill: #68a0b3;font-weight: bold } +.terminal-3503597385-r5 { fill: #868887 } +.terminal-3503597385-r6 { fill: #98a84b;font-weight: bold } +.terminal-3503597385-r7 { fill: #8d7b39 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - Command: ci-image build + Command: ci-image build - + - - -Usage: breeze ci-image build [OPTIONS] - -Build CI image. Include building multiple images for all python versions. - -╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.│ -│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ -│[default: 3.7]                                              │ -│--upgrade-to-newer-dependencies-uWhen set, upgrade all PIP packages to latest.│ -│--upgrade-on-failureWhen set, attempt to run upgrade to newer dependencies when regular build      │ -│fails.                                                                         │ -│--image-tag-tTag the image after building it.(TEXT)[default: latest]│ -│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful│ -│when you build or pull image with --image-tag.                                 │ -│--docker-cache-cCache option for image used during the build.(registry | local | disabled)│ -│[default: registry]                          │ -│--force-buildForce image build no matter if it is determined as needed.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Building images in parallel ────────────────────────────────────────────────────────────────────────────────────────╮ -│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ -│--parallelismMaximum number of processes to use while running the operation in parallel.│ -│(INTEGER RANGE)                                                            │ -│[default: 4; 1<=x<=8]                                                      │ -│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ -│[default: 3.7 3.8 3.9 3.10]                                                   │ -│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ -│--debug-resourcesWhether to show resource information while running in parallel.│ -│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced options (for power users) ─────────────────────────────────────────────────────────────────────────────────╮ -│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)│ -│--install-providers-from-sourcesInstall providers from sources when installing.│ -│--airflow-constraints-modeMode of constraints for CI image building.                             │ -│(constraints-source-providers | constraints | constraints-no-providers)│ -│[default: constraints-source-providers]                                │ -│--airflow-constraints-referenceConstraint reference to use when building the image.(TEXT)│ -│--python-imageIf specified this is the base python image used to build the image. Should be   │ -│something like: python:VERSION-slim-bullseye.                                   │ -│(TEXT)                                                                          │ -│--additional-python-depsAdditional python dependencies to use when building the images.(TEXT)│ -│--additional-extrasAdditional extra package while installing Airflow in the image.(TEXT)│ -│--additional-pip-install-flagsAdditional flags added to `pip install` commands (except reinstalling `pip`     │ -│itself).                                                                        │ -│(TEXT)                                                                          │ -│--additional-dev-apt-depsAdditional apt dev dependencies to use when building the images.(TEXT)│ -│--additional-dev-apt-envAdditional environment variables set when adding dev dependencies.(TEXT)│ -│--additional-dev-apt-commandAdditional command executed before dev apt deps are installed.(TEXT)│ -│--dev-apt-depsApt dev dependencies to use when building the images.(TEXT)│ -│--dev-apt-commandCommand executed before dev apt deps are installed.(TEXT)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ -│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ -│--github-usernameThe user name used to authenticate to GitHub.(TEXT)│ -│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)│ -│--pushPush image after building it.│ -│--empty-imagePrepare empty image tagged with the same name as the Airflow image.│ -│--prepare-buildx-cachePrepares build cache (this is done as separate per-platform steps instead of building the │ -│image).                                                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze ci-image build [OPTIONS] + +Build CI image. Include building multiple images for all python versions. + +╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.7< | 3.8 | 3.9 | 3.10 | 3.11)                           │ +│[default: 3.7]                                              │ +│--upgrade-to-newer-dependencies-uWhen set, upgrade all PIP packages to latest.│ +│--upgrade-on-failureWhen set, attempt to run upgrade to newer dependencies when regular build      │ +│fails.                                                                         │ +│--image-tag-tTag the image after building it.(TEXT)[default: latest]│ +│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful│ +│when you build or pull image with --image-tag.                                 │ +│--docker-cache-cCache option for image used during the build.(registry | local | disabled)│ +│[default: registry]                          │ +│--force-buildForce image build no matter if it is determined as needed.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Building images in parallel ────────────────────────────────────────────────────────────────────────────────────────╮ +│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ +│--parallelismMaximum number of processes to use while running the operation in parallel.│ +│(INTEGER RANGE)                                                            │ +│[default: 4; 1<=x<=8]                                                      │ +│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ +│[default: 3.7 3.8 3.9 3.10 3.11]                                              │ +│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ +│--debug-resourcesWhether to show resource information while running in parallel.│ +│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced options (for power users) ─────────────────────────────────────────────────────────────────────────────────╮ +│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)│ +│--install-providers-from-sourcesInstall providers from sources when installing.│ +│--airflow-constraints-modeMode of constraints for CI image building.                             │ +│(constraints-source-providers | constraints | constraints-no-providers)│ +│[default: constraints-source-providers]                                │ +│--airflow-constraints-referenceConstraint reference to use when building the image.(TEXT)│ +│--python-imageIf specified this is the base python image used to build the image. Should be   │ +│something like: python:VERSION-slim-bullseye.                                   │ +│(TEXT)                                                                          │ +│--additional-python-depsAdditional python dependencies to use when building the images.(TEXT)│ +│--additional-extrasAdditional extra package while installing Airflow in the image.(TEXT)│ +│--additional-pip-install-flagsAdditional flags added to `pip install` commands (except reinstalling `pip`     │ +│itself).                                                                        │ +│(TEXT)                                                                          │ +│--additional-dev-apt-depsAdditional apt dev dependencies to use when building the images.(TEXT)│ +│--additional-dev-apt-envAdditional environment variables set when adding dev dependencies.(TEXT)│ +│--additional-dev-apt-commandAdditional command executed before dev apt deps are installed.(TEXT)│ +│--dev-apt-depsApt dev dependencies to use when building the images.(TEXT)│ +│--dev-apt-commandCommand executed before dev apt deps are installed.(TEXT)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ +│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ +│--github-usernameThe user name used to authenticate to GitHub.(TEXT)│ +│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)│ +│--pushPush image after building it.│ +│--empty-imagePrepare empty image tagged with the same name as the Airflow image.│ +│--prepare-buildx-cachePrepares build cache (this is done as separate per-platform steps instead of building the │ +│image).                                                                                   │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_ci-image_pull.svg b/images/breeze/output_ci-image_pull.svg index abcc06b9b39d5..6f2119c6b3ede 100644 --- a/images/breeze/output_ci-image_pull.svg +++ b/images/breeze/output_ci-image_pull.svg @@ -19,169 +19,169 @@ font-weight: 700; } - .terminal-1876190902-matrix { + .terminal-2343659416-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1876190902-title { + .terminal-2343659416-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1876190902-r1 { fill: #c5c8c6;font-weight: bold } -.terminal-1876190902-r2 { fill: #c5c8c6 } -.terminal-1876190902-r3 { fill: #d0b344;font-weight: bold } -.terminal-1876190902-r4 { fill: #868887 } -.terminal-1876190902-r5 { fill: #68a0b3;font-weight: bold } -.terminal-1876190902-r6 { fill: #98a84b;font-weight: bold } -.terminal-1876190902-r7 { fill: #8d7b39 } + .terminal-2343659416-r1 { fill: #c5c8c6;font-weight: bold } +.terminal-2343659416-r2 { fill: #c5c8c6 } +.terminal-2343659416-r3 { fill: #d0b344;font-weight: bold } +.terminal-2343659416-r4 { fill: #68a0b3;font-weight: bold } +.terminal-2343659416-r5 { fill: #868887 } +.terminal-2343659416-r6 { fill: #98a84b;font-weight: bold } +.terminal-2343659416-r7 { fill: #8d7b39 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - Command: ci-image pull + Command: ci-image pull - + - - -Usage: breeze ci-image pull [OPTIONS] [EXTRA_PYTEST_ARGS]... - -Pull and optionally verify CI images - possibly in parallel for all Python versions. - -╭─ Pull image flags ───────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--image-tag-tTag of the image which is used to pull the image.(TEXT)[default: latest]│ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ -│--verifyVerify image.│ -│--wait-for-imageWait until image is available.│ -│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful when you build│ -│or pull image with --image-tag.                                                               │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Parallel running ───────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ -│--parallelismMaximum number of processes to use while running the operation in parallel.│ -│(INTEGER RANGE)                                                            │ -│[default: 4; 1<=x<=8]                                                      │ -│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ -│[default: 3.7 3.8 3.9 3.10]                                                   │ -│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ -│--debug-resourcesWhether to show resource information while running in parallel.│ -│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze ci-image pull [OPTIONS] [EXTRA_PYTEST_ARGS]... + +Pull and optionally verify CI images - possibly in parallel for all Python versions. + +╭─ Pull image flags ───────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--image-tag-tTag of the image which is used to pull the image.(TEXT)[default: latest]│ +│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10 | 3.11)│ +│[default: 3.7]                                              │ +│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ +│--verifyVerify image.│ +│--wait-for-imageWait until image is available.│ +│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful when you build│ +│or pull image with --image-tag.                                                               │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Parallel running ───────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ +│--parallelismMaximum number of processes to use while running the operation in parallel.│ +│(INTEGER RANGE)                                                            │ +│[default: 4; 1<=x<=8]                                                      │ +│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ +│[default: 3.7 3.8 3.9 3.10 3.11]                                              │ +│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ +│--debug-resourcesWhether to show resource information while running in parallel.│ +│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_ci-image_verify.svg b/images/breeze/output_ci-image_verify.svg index 3b7d4be25b787..ee3b030158e60 100644 --- a/images/breeze/output_ci-image_verify.svg +++ b/images/breeze/output_ci-image_verify.svg @@ -19,113 +19,113 @@ font-weight: 700; } - .terminal-4266854521-matrix { + .terminal-3263695128-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-4266854521-title { + .terminal-3263695128-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-4266854521-r1 { fill: #c5c8c6;font-weight: bold } -.terminal-4266854521-r2 { fill: #c5c8c6 } -.terminal-4266854521-r3 { fill: #d0b344;font-weight: bold } -.terminal-4266854521-r4 { fill: #868887 } -.terminal-4266854521-r5 { fill: #68a0b3;font-weight: bold } -.terminal-4266854521-r6 { fill: #98a84b;font-weight: bold } -.terminal-4266854521-r7 { fill: #8d7b39 } + .terminal-3263695128-r1 { fill: #c5c8c6;font-weight: bold } +.terminal-3263695128-r2 { fill: #c5c8c6 } +.terminal-3263695128-r3 { fill: #d0b344;font-weight: bold } +.terminal-3263695128-r4 { fill: #68a0b3;font-weight: bold } +.terminal-3263695128-r5 { fill: #868887 } +.terminal-3263695128-r6 { fill: #98a84b;font-weight: bold } +.terminal-3263695128-r7 { fill: #8d7b39 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - Command: ci-image verify + Command: ci-image verify - + - - -Usage: breeze ci-image verify [OPTIONS] [EXTRA_PYTEST_ARGS]... - -Verify CI image. - -╭─ Verify image flags ─────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--image-name-nName of the image to verify (overrides --python and --image-tag).(TEXT)│ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--image-tag-tTag of the image when verifying it.(TEXT)[default: latest]│ -│--pullPull image is missing before attempting to verify it.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze ci-image verify [OPTIONS] [EXTRA_PYTEST_ARGS]... + +Verify CI image. + +╭─ Verify image flags ─────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--image-name-nName of the image to verify (overrides --python and --image-tag).(TEXT)│ +│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10 | 3.11)│ +│[default: 3.7]                                              │ +│--image-tag-tTag of the image when verifying it.(TEXT)[default: latest]│ +│--pullPull image is missing before attempting to verify it.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_ci.svg b/images/breeze/output_ci.svg index ad5ff72659076..601f54e29ac89 100644 --- a/images/breeze/output_ci.svg +++ b/images/breeze/output_ci.svg @@ -35,8 +35,8 @@ .terminal-1385492439-r1 { fill: #c5c8c6;font-weight: bold } .terminal-1385492439-r2 { fill: #c5c8c6 } .terminal-1385492439-r3 { fill: #d0b344;font-weight: bold } -.terminal-1385492439-r4 { fill: #868887 } -.terminal-1385492439-r5 { fill: #68a0b3;font-weight: bold } +.terminal-1385492439-r4 { fill: #68a0b3;font-weight: bold } +.terminal-1385492439-r5 { fill: #868887 } .terminal-1385492439-r6 { fill: #98a84b;font-weight: bold } @@ -105,22 +105,22 @@ -Usage: breeze ci [OPTIONS] COMMAND [ARGS]... +Usage: breeze ci [OPTIONS] COMMAND [ARGS]... -Tools that CI workflows use to cleanup/manage CI environment +Tools that CI workflows use to cleanup/manage CI environment -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ CI commands ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│fix-ownership          Fix ownership of source files to be same as host user.                                     │ -│free-space             Free space for jobs run in CI.                                                             │ -│resource-check         Check if available docker resources are enough.                                            │ -│selective-check        Checks what kind of tests should be run for an incoming commit.                            │ -│find-newer-dependenciesFinds which dependencies are being upgraded.                                               │ -│get-workflow-info      Retrieve information about current workflow in the CIand produce github actions output     │ -│extracted from it.                                                                         │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ CI commands ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│fix-ownership          Fix ownership of source files to be same as host user.                                     │ +│free-space             Free space for jobs run in CI.                                                             │ +│resource-check         Check if available docker resources are enough.                                            │ +│selective-check        Checks what kind of tests should be run for an incoming commit.                            │ +│find-newer-dependenciesFinds which dependencies are being upgraded.                                               │ +│get-workflow-info      Retrieve information about current workflow in the CIand produce github actions output     │ +│extracted from it.                                                                         │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_ci_find-newer-dependencies.svg b/images/breeze/output_ci_find-newer-dependencies.svg index 95bba2a0b9b54..43029e436138a 100644 --- a/images/breeze/output_ci_find-newer-dependencies.svg +++ b/images/breeze/output_ci_find-newer-dependencies.svg @@ -19,121 +19,121 @@ font-weight: 700; } - .terminal-1646402681-matrix { + .terminal-3800571160-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1646402681-title { + .terminal-3800571160-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1646402681-r1 { fill: #c5c8c6;font-weight: bold } -.terminal-1646402681-r2 { fill: #c5c8c6 } -.terminal-1646402681-r3 { fill: #d0b344;font-weight: bold } -.terminal-1646402681-r4 { fill: #868887 } -.terminal-1646402681-r5 { fill: #68a0b3;font-weight: bold } -.terminal-1646402681-r6 { fill: #98a84b;font-weight: bold } -.terminal-1646402681-r7 { fill: #8d7b39 } + .terminal-3800571160-r1 { fill: #c5c8c6;font-weight: bold } +.terminal-3800571160-r2 { fill: #c5c8c6 } +.terminal-3800571160-r3 { fill: #d0b344;font-weight: bold } +.terminal-3800571160-r4 { fill: #68a0b3;font-weight: bold } +.terminal-3800571160-r5 { fill: #868887 } +.terminal-3800571160-r6 { fill: #98a84b;font-weight: bold } +.terminal-3800571160-r7 { fill: #8d7b39 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - Command: ci find-newer-dependencies + Command: ci find-newer-dependencies - + - - -Usage: breeze ci find-newer-dependencies [OPTIONS] - -Finds which dependencies are being upgraded. - -╭─ Find newer dependencies flags ──────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.│ -│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ -│[default: 3.7]                                              │ -│--timezoneTimezone to use during the check.(TEXT)│ -│--airflow-constraints-referenceConstraint reference to use. Useful with --use-airflow-version parameter to    │ -│specify constraints for the installed version and to find newer dependencies   │ -│(TEXT)                                                                         │ -│--updated-on-or-afterDate when the release was updated after.(TEXT)│ -│--max-ageMax age of the last release (used if no updated-on-or-after if specified).│ -│(INTEGER)                                                                 │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze ci find-newer-dependencies [OPTIONS] + +Finds which dependencies are being upgraded. + +╭─ Find newer dependencies flags ──────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.7< | 3.8 | 3.9 | 3.10 | 3.11)                           │ +│[default: 3.7]                                              │ +│--timezoneTimezone to use during the check.(TEXT)│ +│--airflow-constraints-referenceConstraint reference to use. Useful with --use-airflow-version parameter to    │ +│specify constraints for the installed version and to find newer dependencies   │ +│(TEXT)                                                                         │ +│--updated-on-or-afterDate when the release was updated after.(TEXT)│ +│--max-ageMax age of the last release (used if no updated-on-or-after if specified).│ +│(INTEGER)                                                                 │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_k8s.svg b/images/breeze/output_k8s.svg index 9b6e868d5d48b..d9e05970f4039 100644 --- a/images/breeze/output_k8s.svg +++ b/images/breeze/output_k8s.svg @@ -35,8 +35,8 @@ .terminal-781963109-r1 { fill: #c5c8c6;font-weight: bold } .terminal-781963109-r2 { fill: #c5c8c6 } .terminal-781963109-r3 { fill: #d0b344;font-weight: bold } -.terminal-781963109-r4 { fill: #868887 } -.terminal-781963109-r5 { fill: #68a0b3;font-weight: bold } +.terminal-781963109-r4 { fill: #68a0b3;font-weight: bold } +.terminal-781963109-r5 { fill: #868887 } .terminal-781963109-r6 { fill: #98a84b;font-weight: bold } @@ -147,36 +147,36 @@ -Usage: breeze k8s [OPTIONS] COMMAND [ARGS]... +Usage: breeze k8s [OPTIONS] COMMAND [ARGS]... Tools that developers use to run Kubernetes tests -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ K8S cluster management commands ────────────────────────────────────────────────────────────────────────────────────╮ -│setup-env        Setup shared Kubernetes virtual environment and tools.                                           │ -│create-cluster   Create a KinD Cluster for Python and Kubernetes version specified (optionally create all clusters│ -│in parallel).                                                                                    │ -│configure-clusterConfigures cluster for airflow deployment - creates namespaces and test resources (optionally for│ -│all clusters in parallel).                                                                       │ -│build-k8s-image  Build k8s-ready airflow image (optionally all images in parallel).                               │ -│upload-k8s-image Upload k8s-ready airflow image to the KinD cluster (optionally to all clusters in parallel)      │ -│deploy-airflow   Deploy airflow image to the current KinD cluster (or all clusters).                              │ -│delete-cluster   Delete the current KinD Cluster (optionally all clusters).                                       │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ K8S inspection commands ────────────────────────────────────────────────────────────────────────────────────────────╮ -│status  Check status of the current cluster and airflow deployed to it (optionally all clusters).                 │ -│logs    Dump k8s logs to ${TMP_DIR}/kind_logs_<cluster_name> directory (optionally all clusters).                 │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ K8S testing commands ───────────────────────────────────────────────────────────────────────────────────────────────╮ -│tests             Run tests against the current KinD cluster (optionally for all clusters in parallel).           │ -│run-complete-testsRun complete k8s tests consisting of: creating cluster, building and uploading image, deploying │ -│airflow, running tests and deleting clusters (optionally for all clusters in parallel).         │ -│shell             Run shell environment for the current KinD cluster.                                             │ -│k9s               Run k9s tool. You can pass any k9s args as extra args.                                          │ -│logs              Dump k8s logs to ${TMP_DIR}/kind_logs_<cluster_name> directory (optionally all clusters).       │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ K8S cluster management commands ────────────────────────────────────────────────────────────────────────────────────╮ +│setup-env        Setup shared Kubernetes virtual environment and tools.                                           │ +│create-cluster   Create a KinD Cluster for Python and Kubernetes version specified (optionally create all clusters│ +│in parallel).                                                                                    │ +│configure-clusterConfigures cluster for airflow deployment - creates namespaces and test resources (optionally for│ +│all clusters in parallel).                                                                       │ +│build-k8s-image  Build k8s-ready airflow image (optionally all images in parallel).                               │ +│upload-k8s-image Upload k8s-ready airflow image to the KinD cluster (optionally to all clusters in parallel)      │ +│deploy-airflow   Deploy airflow image to the current KinD cluster (or all clusters).                              │ +│delete-cluster   Delete the current KinD Cluster (optionally all clusters).                                       │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ K8S inspection commands ────────────────────────────────────────────────────────────────────────────────────────────╮ +│status  Check status of the current cluster and airflow deployed to it (optionally all clusters).                 │ +│logs    Dump k8s logs to ${TMP_DIR}/kind_logs_<cluster_name> directory (optionally all clusters).                 │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ K8S testing commands ───────────────────────────────────────────────────────────────────────────────────────────────╮ +│tests             Run tests against the current KinD cluster (optionally for all clusters in parallel).           │ +│run-complete-testsRun complete k8s tests consisting of: creating cluster, building and uploading image, deploying │ +│airflow, running tests and deleting clusters (optionally for all clusters in parallel).         │ +│shell             Run shell environment for the current KinD cluster.                                             │ +│k9s               Run k9s tool. You can pass any k9s args as extra args.                                          │ +│logs              Dump k8s logs to ${TMP_DIR}/kind_logs_<cluster_name> directory (optionally all clusters).       │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_k8s_build-k8s-image.svg b/images/breeze/output_k8s_build-k8s-image.svg index d34d962cc977d..1422f1dc77822 100644 --- a/images/breeze/output_k8s_build-k8s-image.svg +++ b/images/breeze/output_k8s_build-k8s-image.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - Command: k8s build-k8s-image + Command: k8s build-k8s-image - + - - -Usage: breeze k8s build-k8s-image [OPTIONS] - -Build k8s-ready airflow image (optionally all images in parallel). - -╭─ Build image flags ──────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--rebuild-base-imageRebuilds base Airflow image before building K8S image.│ -│--image-tag-tImage tag used to build K8S image from.(TEXT)[default: latest]│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Parallel options ───────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ -│--parallelismMaximum number of processes to use while running the operation in parallel.│ -│(INTEGER RANGE)                                                            │ -│[default: 4; 1<=x<=8]                                                      │ -│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ -│[default: 3.7 3.8 3.9 3.10]                                                   │ -│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ -│--debug-resourcesWhether to show resource information while running in parallel.│ -│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze k8s build-k8s-image [OPTIONS] + +Build k8s-ready airflow image (optionally all images in parallel). + +╭─ Build image flags ──────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.7< | 3.8 | 3.9 | 3.10 | 3.11)                           │ +│[default: 3.7]                                              │ +│--rebuild-base-imageRebuilds base Airflow image before building K8S image.│ +│--image-tag-tImage tag used to build K8S image from.(TEXT)[default: latest]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Parallel options ───────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ +│--parallelismMaximum number of processes to use while running the operation in parallel.│ +│(INTEGER RANGE)                                                            │ +│[default: 4; 1<=x<=8]                                                      │ +│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ +│[default: 3.7 3.8 3.9 3.10 3.11]                                              │ +│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ +│--debug-resourcesWhether to show resource information while running in parallel.│ +│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_k8s_configure-cluster.svg b/images/breeze/output_k8s_configure-cluster.svg index 761b21aebeb89..97543ad6eff05 100644 --- a/images/breeze/output_k8s_configure-cluster.svg +++ b/images/breeze/output_k8s_configure-cluster.svg @@ -19,157 +19,157 @@ font-weight: 700; } - .terminal-2822430757-matrix { + .terminal-2651709703-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2822430757-title { + .terminal-2651709703-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2822430757-r1 { fill: #c5c8c6;font-weight: bold } -.terminal-2822430757-r2 { fill: #c5c8c6 } -.terminal-2822430757-r3 { fill: #d0b344;font-weight: bold } -.terminal-2822430757-r4 { fill: #868887 } -.terminal-2822430757-r5 { fill: #68a0b3;font-weight: bold } -.terminal-2822430757-r6 { fill: #98a84b;font-weight: bold } -.terminal-2822430757-r7 { fill: #8d7b39 } + .terminal-2651709703-r1 { fill: #c5c8c6;font-weight: bold } +.terminal-2651709703-r2 { fill: #c5c8c6 } +.terminal-2651709703-r3 { fill: #d0b344;font-weight: bold } +.terminal-2651709703-r4 { fill: #68a0b3;font-weight: bold } +.terminal-2651709703-r5 { fill: #868887 } +.terminal-2651709703-r6 { fill: #98a84b;font-weight: bold } +.terminal-2651709703-r7 { fill: #8d7b39 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - Command: k8s configure-cluster + Command: k8s configure-cluster - + - - -Usage: breeze k8s configure-cluster [OPTIONS] - -Configures cluster for airflow deployment - creates namespaces and test resources (optionally for all clusters in  -parallel). - -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.│ -│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ -│[default: 3.7]                                              │ -│--kubernetes-versionKubernetes version used to create the KinD cluster of.│ -│(>v1.25.2< | v1.24.6 | v1.23.12 | v1.22.15 | v1.21.14)│ -│[default: v1.25.2]                                    │ -│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ -│--parallelismMaximum number of processes to use while running the operation in parallel for       │ -│cluster operations.                                                                  │ -│(INTEGER RANGE)                                                                      │ -│[default: 2; 1<=x<=4]                                                                │ -│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ -│--debug-resourcesWhether to show resource information while running in parallel.│ -│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ -│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ -│[default: 3.7 3.8 3.9 3.10]                                                   │ -│--kubernetes-versionsKubernetes versions used to run in parallel (space separated).(TEXT)│ -│[default: v1.25.2 v1.24.6 v1.23.12 v1.22.15 v1.21.14]         │ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze k8s configure-cluster [OPTIONS] + +Configures cluster for airflow deployment - creates namespaces and test resources (optionally for all clusters in  +parallel). + +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.7< | 3.8 | 3.9 | 3.10 | 3.11)                           │ +│[default: 3.7]                                              │ +│--kubernetes-versionKubernetes version used to create the KinD cluster of.│ +│(>v1.25.2< | v1.24.6 | v1.23.12 | v1.22.15 | v1.21.14)│ +│[default: v1.25.2]                                    │ +│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ +│--parallelismMaximum number of processes to use while running the operation in parallel for       │ +│cluster operations.                                                                  │ +│(INTEGER RANGE)                                                                      │ +│[default: 2; 1<=x<=4]                                                                │ +│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ +│--debug-resourcesWhether to show resource information while running in parallel.│ +│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ +│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ +│[default: 3.7 3.8 3.9 3.10 3.11]                                              │ +│--kubernetes-versionsKubernetes versions used to run in parallel (space separated).(TEXT)│ +│[default: v1.25.2 v1.24.6 v1.23.12 v1.22.15 v1.21.14]         │ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_k8s_create-cluster.svg b/images/breeze/output_k8s_create-cluster.svg index da407be401306..df39cea739ffd 100644 --- a/images/breeze/output_k8s_create-cluster.svg +++ b/images/breeze/output_k8s_create-cluster.svg @@ -19,173 +19,173 @@ font-weight: 700; } - .terminal-3311949748-matrix { + .terminal-660625558-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3311949748-title { + .terminal-660625558-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3311949748-r1 { fill: #c5c8c6;font-weight: bold } -.terminal-3311949748-r2 { fill: #c5c8c6 } -.terminal-3311949748-r3 { fill: #d0b344;font-weight: bold } -.terminal-3311949748-r4 { fill: #868887 } -.terminal-3311949748-r5 { fill: #68a0b3;font-weight: bold } -.terminal-3311949748-r6 { fill: #98a84b;font-weight: bold } -.terminal-3311949748-r7 { fill: #8d7b39 } + .terminal-660625558-r1 { fill: #c5c8c6;font-weight: bold } +.terminal-660625558-r2 { fill: #c5c8c6 } +.terminal-660625558-r3 { fill: #d0b344;font-weight: bold } +.terminal-660625558-r4 { fill: #68a0b3;font-weight: bold } +.terminal-660625558-r5 { fill: #868887 } +.terminal-660625558-r6 { fill: #98a84b;font-weight: bold } +.terminal-660625558-r7 { fill: #8d7b39 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - Command: k8s create-cluster + Command: k8s create-cluster - + - - -Usage: breeze k8s create-cluster [OPTIONS] - -Create a KinD Cluster for Python and Kubernetes version specified (optionally create all clusters in parallel). - -╭─ K8S cluster creation flags ─────────────────────────────────────────────────────────────────────────────────────────╮ -│--force-recreate-clusterForce recreation of the cluster even if it is already created.│ -│--python-pPython major/minor version used in Airflow image for images.│ -│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ -│[default: 3.7]                                              │ -│--kubernetes-versionKubernetes version used to create the KinD cluster of.│ -│(>v1.25.2< | v1.24.6 | v1.23.12 | v1.22.15 | v1.21.14)│ -│[default: v1.25.2]                                    │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Parallel options ───────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ -│--parallelismMaximum number of processes to use while running the operation in parallel for cluster │ -│operations.                                                                            │ -│(INTEGER RANGE)                                                                        │ -│[default: 2; 1<=x<=4]                                                                  │ -│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ -│[default: 3.7 3.8 3.9 3.10]                                                   │ -│--kubernetes-versionsKubernetes versions used to run in parallel (space separated).(TEXT)│ -│[default: v1.25.2 v1.24.6 v1.23.12 v1.22.15 v1.21.14]         │ -│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ -│--debug-resourcesWhether to show resource information while running in parallel.│ -│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze k8s create-cluster [OPTIONS] + +Create a KinD Cluster for Python and Kubernetes version specified (optionally create all clusters in parallel). + +╭─ K8S cluster creation flags ─────────────────────────────────────────────────────────────────────────────────────────╮ +│--force-recreate-clusterForce recreation of the cluster even if it is already created.│ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.7< | 3.8 | 3.9 | 3.10 | 3.11)                           │ +│[default: 3.7]                                              │ +│--kubernetes-versionKubernetes version used to create the KinD cluster of.│ +│(>v1.25.2< | v1.24.6 | v1.23.12 | v1.22.15 | v1.21.14)│ +│[default: v1.25.2]                                    │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Parallel options ───────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ +│--parallelismMaximum number of processes to use while running the operation in parallel for cluster │ +│operations.                                                                            │ +│(INTEGER RANGE)                                                                        │ +│[default: 2; 1<=x<=4]                                                                  │ +│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ +│[default: 3.7 3.8 3.9 3.10 3.11]                                              │ +│--kubernetes-versionsKubernetes versions used to run in parallel (space separated).(TEXT)│ +│[default: v1.25.2 v1.24.6 v1.23.12 v1.22.15 v1.21.14]         │ +│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ +│--debug-resourcesWhether to show resource information while running in parallel.│ +│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_k8s_delete-cluster.svg b/images/breeze/output_k8s_delete-cluster.svg index aa0ddab2db150..9225509e06ef3 100644 --- a/images/breeze/output_k8s_delete-cluster.svg +++ b/images/breeze/output_k8s_delete-cluster.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - Command: k8s delete-cluster + Command: k8s delete-cluster - + - - -Usage: breeze k8s delete-cluster [OPTIONS] - -Delete the current KinD Cluster (optionally all clusters). - -╭─ K8S cluster delete flags ───────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--kubernetes-versionKubernetes version used to create the KinD cluster of.│ -│(>v1.25.2< | v1.24.6 | v1.23.12 | v1.22.15 | v1.21.14)│ -│[default: v1.25.2]                                    │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ K8S multi-cluster flags ────────────────────────────────────────────────────────────────────────────────────────────╮ -│--allApply it to all created clusters│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze k8s delete-cluster [OPTIONS] + +Delete the current KinD Cluster (optionally all clusters). + +╭─ K8S cluster delete flags ───────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.7< | 3.8 | 3.9 | 3.10 | 3.11)                           │ +│[default: 3.7]                                              │ +│--kubernetes-versionKubernetes version used to create the KinD cluster of.│ +│(>v1.25.2< | v1.24.6 | v1.23.12 | v1.22.15 | v1.21.14)│ +│[default: v1.25.2]                                    │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ K8S multi-cluster flags ────────────────────────────────────────────────────────────────────────────────────────────╮ +│--allApply it to all created clusters│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_k8s_deploy-airflow.svg b/images/breeze/output_k8s_deploy-airflow.svg index 5ba02f94a4dea..851576e7c4856 100644 --- a/images/breeze/output_k8s_deploy-airflow.svg +++ b/images/breeze/output_k8s_deploy-airflow.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - Command: k8s deploy-airflow + Command: k8s deploy-airflow - + - - -Usage: breeze k8s deploy-airflow [OPTIONS] [EXTRA_OPTIONS]... - -Deploy airflow image to the current KinD cluster (or all clusters). - -╭─ Airflow deploy flags ───────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--kubernetes-versionKubernetes version used to create the KinD cluster of.│ -│(>v1.25.2< | v1.24.6 | v1.23.12 | v1.22.15 | v1.21.14)│ -│[default: v1.25.2]                                    │ -│--executorExecutor to use for a kubernetes cluster.                                         │ -│(>KubernetesExecutor< | CeleryExecutor | LocalExecutor | CeleryKubernetesExecutor)│ -│[default: KubernetesExecutor]                                                     │ -│--upgradeUpgrade Helm Chart rather than installing it.│ -│--wait-time-in-secondsWait for Airflow webserver for specified number of seconds.(INTEGER RANGE)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Parallel options ───────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ -│--parallelismMaximum number of processes to use while running the operation in parallel for cluster │ -│operations.                                                                            │ -│(INTEGER RANGE)                                                                        │ -│[default: 2; 1<=x<=4]                                                                  │ -│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ -│[default: 3.7 3.8 3.9 3.10]                                                   │ -│--kubernetes-versionsKubernetes versions used to run in parallel (space separated).(TEXT)│ -│[default: v1.25.2 v1.24.6 v1.23.12 v1.22.15 v1.21.14]         │ -│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ -│--debug-resourcesWhether to show resource information while running in parallel.│ -│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze k8s deploy-airflow [OPTIONS] [EXTRA_OPTIONS]... + +Deploy airflow image to the current KinD cluster (or all clusters). + +╭─ Airflow deploy flags ───────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.7< | 3.8 | 3.9 | 3.10 | 3.11)                           │ +│[default: 3.7]                                              │ +│--kubernetes-versionKubernetes version used to create the KinD cluster of.│ +│(>v1.25.2< | v1.24.6 | v1.23.12 | v1.22.15 | v1.21.14)│ +│[default: v1.25.2]                                    │ +│--executorExecutor to use for a kubernetes cluster.                                         │ +│(>KubernetesExecutor< | CeleryExecutor | LocalExecutor | CeleryKubernetesExecutor)│ +│[default: KubernetesExecutor]                                                     │ +│--upgradeUpgrade Helm Chart rather than installing it.│ +│--wait-time-in-secondsWait for Airflow webserver for specified number of seconds.(INTEGER RANGE)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Parallel options ───────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ +│--parallelismMaximum number of processes to use while running the operation in parallel for cluster │ +│operations.                                                                            │ +│(INTEGER RANGE)                                                                        │ +│[default: 2; 1<=x<=4]                                                                  │ +│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ +│[default: 3.7 3.8 3.9 3.10 3.11]                                              │ +│--kubernetes-versionsKubernetes versions used to run in parallel (space separated).(TEXT)│ +│[default: v1.25.2 v1.24.6 v1.23.12 v1.22.15 v1.21.14]         │ +│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ +│--debug-resourcesWhether to show resource information while running in parallel.│ +│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_k8s_k9s.svg b/images/breeze/output_k8s_k9s.svg index 54af3bb8c5822..dfdb2f01f0561 100644 --- a/images/breeze/output_k8s_k9s.svg +++ b/images/breeze/output_k8s_k9s.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - Command: k8s k9s + Command: k8s k9s - + - - -Usage: breeze k8s k9s [OPTIONS] [K9S_ARGS]... - -Run k9s tool. You can pass any k9s args as extra args. - -╭─ K8S k9s flags ──────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--kubernetes-versionKubernetes version used to create the KinD cluster of.│ -│(>v1.25.2< | v1.24.6 | v1.23.12 | v1.22.15 | v1.21.14)│ -│[default: v1.25.2]                                    │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze k8s k9s [OPTIONS] [K9S_ARGS]... + +Run k9s tool. You can pass any k9s args as extra args. + +╭─ K8S k9s flags ──────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.7< | 3.8 | 3.9 | 3.10 | 3.11)                           │ +│[default: 3.7]                                              │ +│--kubernetes-versionKubernetes version used to create the KinD cluster of.│ +│(>v1.25.2< | v1.24.6 | v1.23.12 | v1.22.15 | v1.21.14)│ +│[default: v1.25.2]                                    │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_k8s_logs.svg b/images/breeze/output_k8s_logs.svg index 6d03a62388dd8..98f777b2d5190 100644 --- a/images/breeze/output_k8s_logs.svg +++ b/images/breeze/output_k8s_logs.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - Command: k8s logs + Command: k8s logs - + - - -Usage: breeze k8s logs [OPTIONS] - -Dump k8s logs to ${TMP_DIR}/kind_logs_<cluster_name> directory (optionally all clusters). - -╭─ K8S logs flags ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--kubernetes-versionKubernetes version used to create the KinD cluster of.│ -│(>v1.25.2< | v1.24.6 | v1.23.12 | v1.22.15 | v1.21.14)│ -│[default: v1.25.2]                                    │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ K8S multi-cluster flags ────────────────────────────────────────────────────────────────────────────────────────────╮ -│--allApply it to all created clusters│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze k8s logs [OPTIONS] + +Dump k8s logs to ${TMP_DIR}/kind_logs_<cluster_name> directory (optionally all clusters). + +╭─ K8S logs flags ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.7< | 3.8 | 3.9 | 3.10 | 3.11)                           │ +│[default: 3.7]                                              │ +│--kubernetes-versionKubernetes version used to create the KinD cluster of.│ +│(>v1.25.2< | v1.24.6 | v1.23.12 | v1.22.15 | v1.21.14)│ +│[default: v1.25.2]                                    │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ K8S multi-cluster flags ────────────────────────────────────────────────────────────────────────────────────────────╮ +│--allApply it to all created clusters│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_k8s_run-complete-tests.svg b/images/breeze/output_k8s_run-complete-tests.svg index 86640e978872c..60785508537f7 100644 --- a/images/breeze/output_k8s_run-complete-tests.svg +++ b/images/breeze/output_k8s_run-complete-tests.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - Command: k8s run-complete-tests + Command: k8s run-complete-tests - + - - -Usage: breeze k8s run-complete-tests [OPTIONS] [TEST_ARGS]... - -Run complete k8s tests consisting of: creating cluster, building and uploading image, deploying airflow, running tests -and deleting clusters (optionally for all clusters in parallel). - -╭─ K8S cluster creation flags ─────────────────────────────────────────────────────────────────────────────────────────╮ -│--force-recreate-clusterForce recreation of the cluster even if it is already created.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Airflow deploy flags ───────────────────────────────────────────────────────────────────────────────────────────────╮ -│--upgradeUpgrade Helm Chart rather than installing it.│ -│--wait-time-in-secondsWait for Airflow webserver for specified number of seconds.(INTEGER RANGE)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Build image flags ──────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--rebuild-base-imageRebuilds base Airflow image before building K8S image.│ -│--image-tag-tImage tag used to build K8S image from.(TEXT)[default: latest]│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ K8S tests flags ────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--kubernetes-versionKubernetes version used to create the KinD cluster of.│ -│(>v1.25.2< | v1.24.6 | v1.23.12 | v1.22.15 | v1.21.14)│ -│[default: v1.25.2]                                    │ -│--executorExecutor to use for a kubernetes cluster.                                         │ -│(>KubernetesExecutor< | CeleryExecutor | LocalExecutor | CeleryKubernetesExecutor)│ -│[default: KubernetesExecutor]                                                     │ -│--force-venv-setupForce recreation of the virtualenv.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Parallel options ───────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ -│--parallelismMaximum number of processes to use while running the operation in parallel for cluster │ -│operations.                                                                            │ -│(INTEGER RANGE)                                                                        │ -│[default: 2; 1<=x<=4]                                                                  │ -│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ -│[default: 3.7 3.8 3.9 3.10]                                                   │ -│--kubernetes-versionsKubernetes versions used to run in parallel (space separated).(TEXT)│ -│[default: v1.25.2 v1.24.6 v1.23.12 v1.22.15 v1.21.14]         │ -│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ -│--debug-resourcesWhether to show resource information while running in parallel.│ -│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze k8s run-complete-tests [OPTIONS] [TEST_ARGS]... + +Run complete k8s tests consisting of: creating cluster, building and uploading image, deploying airflow, running tests +and deleting clusters (optionally for all clusters in parallel). + +╭─ K8S cluster creation flags ─────────────────────────────────────────────────────────────────────────────────────────╮ +│--force-recreate-clusterForce recreation of the cluster even if it is already created.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Airflow deploy flags ───────────────────────────────────────────────────────────────────────────────────────────────╮ +│--upgradeUpgrade Helm Chart rather than installing it.│ +│--wait-time-in-secondsWait for Airflow webserver for specified number of seconds.(INTEGER RANGE)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Build image flags ──────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--rebuild-base-imageRebuilds base Airflow image before building K8S image.│ +│--image-tag-tImage tag used to build K8S image from.(TEXT)[default: latest]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ K8S tests flags ────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.7< | 3.8 | 3.9 | 3.10 | 3.11)                           │ +│[default: 3.7]                                              │ +│--kubernetes-versionKubernetes version used to create the KinD cluster of.│ +│(>v1.25.2< | v1.24.6 | v1.23.12 | v1.22.15 | v1.21.14)│ +│[default: v1.25.2]                                    │ +│--executorExecutor to use for a kubernetes cluster.                                         │ +│(>KubernetesExecutor< | CeleryExecutor | LocalExecutor | CeleryKubernetesExecutor)│ +│[default: KubernetesExecutor]                                                     │ +│--force-venv-setupForce recreation of the virtualenv.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Parallel options ───────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ +│--parallelismMaximum number of processes to use while running the operation in parallel for cluster │ +│operations.                                                                            │ +│(INTEGER RANGE)                                                                        │ +│[default: 2; 1<=x<=4]                                                                  │ +│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ +│[default: 3.7 3.8 3.9 3.10 3.11]                                              │ +│--kubernetes-versionsKubernetes versions used to run in parallel (space separated).(TEXT)│ +│[default: v1.25.2 v1.24.6 v1.23.12 v1.22.15 v1.21.14]         │ +│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ +│--debug-resourcesWhether to show resource information while running in parallel.│ +│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_k8s_shell.svg b/images/breeze/output_k8s_shell.svg index 72affc47906e3..33e99f9580015 100644 --- a/images/breeze/output_k8s_shell.svg +++ b/images/breeze/output_k8s_shell.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - Command: k8s shell + Command: k8s shell - + - - -Usage: breeze k8s shell [OPTIONS] [SHELL_ARGS]... - -Run shell environment for the current KinD cluster. - -╭─ K8S shell flags ────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--kubernetes-versionKubernetes version used to create the KinD cluster of.│ -│(>v1.25.2< | v1.24.6 | v1.23.12 | v1.22.15 | v1.21.14)│ -│[default: v1.25.2]                                    │ -│--executorExecutor to use for a kubernetes cluster.                                         │ -│(>KubernetesExecutor< | CeleryExecutor | LocalExecutor | CeleryKubernetesExecutor)│ -│[default: KubernetesExecutor]                                                     │ -│--force-venv-setupForce recreation of the virtualenv.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze k8s shell [OPTIONS] [SHELL_ARGS]... + +Run shell environment for the current KinD cluster. + +╭─ K8S shell flags ────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.7< | 3.8 | 3.9 | 3.10 | 3.11)                           │ +│[default: 3.7]                                              │ +│--kubernetes-versionKubernetes version used to create the KinD cluster of.│ +│(>v1.25.2< | v1.24.6 | v1.23.12 | v1.22.15 | v1.21.14)│ +│[default: v1.25.2]                                    │ +│--executorExecutor to use for a kubernetes cluster.                                         │ +│(>KubernetesExecutor< | CeleryExecutor | LocalExecutor | CeleryKubernetesExecutor)│ +│[default: KubernetesExecutor]                                                     │ +│--force-venv-setupForce recreation of the virtualenv.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_k8s_status.svg b/images/breeze/output_k8s_status.svg index 894a5d2956ccc..fae2eef401d79 100644 --- a/images/breeze/output_k8s_status.svg +++ b/images/breeze/output_k8s_status.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - Command: k8s status + Command: k8s status - + - - -Usage: breeze k8s status [OPTIONS] - -Check status of the current cluster and airflow deployed to it (optionally all clusters). - -╭─ K8S cluster status flags ───────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--kubernetes-versionKubernetes version used to create the KinD cluster of.│ -│(>v1.25.2< | v1.24.6 | v1.23.12 | v1.22.15 | v1.21.14)│ -│[default: v1.25.2]                                    │ -│--wait-time-in-secondsWait for Airflow webserver for specified number of seconds.(INTEGER RANGE)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ K8S multi-cluster flags ────────────────────────────────────────────────────────────────────────────────────────────╮ -│--allApply it to all created clusters│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze k8s status [OPTIONS] + +Check status of the current cluster and airflow deployed to it (optionally all clusters). + +╭─ K8S cluster status flags ───────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.7< | 3.8 | 3.9 | 3.10 | 3.11)                           │ +│[default: 3.7]                                              │ +│--kubernetes-versionKubernetes version used to create the KinD cluster of.│ +│(>v1.25.2< | v1.24.6 | v1.23.12 | v1.22.15 | v1.21.14)│ +│[default: v1.25.2]                                    │ +│--wait-time-in-secondsWait for Airflow webserver for specified number of seconds.(INTEGER RANGE)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ K8S multi-cluster flags ────────────────────────────────────────────────────────────────────────────────────────────╮ +│--allApply it to all created clusters│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_k8s_tests.svg b/images/breeze/output_k8s_tests.svg index 724be6f953c4c..6b4e11fcfad9c 100644 --- a/images/breeze/output_k8s_tests.svg +++ b/images/breeze/output_k8s_tests.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - Command: k8s tests + Command: k8s tests - + - - -Usage: breeze k8s tests [OPTIONS] [TEST_ARGS]... - -Run tests against the current KinD cluster (optionally for all clusters in parallel). - -╭─ K8S tests flags ────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--kubernetes-versionKubernetes version used to create the KinD cluster of.│ -│(>v1.25.2< | v1.24.6 | v1.23.12 | v1.22.15 | v1.21.14)│ -│[default: v1.25.2]                                    │ -│--executorExecutor to use for a kubernetes cluster.                                         │ -│(>KubernetesExecutor< | CeleryExecutor | LocalExecutor | CeleryKubernetesExecutor)│ -│[default: KubernetesExecutor]                                                     │ -│--force-venv-setupForce recreation of the virtualenv.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Parallel options ───────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ -│--parallelismMaximum number of processes to use while running the operation in parallel for cluster │ -│operations.                                                                            │ -│(INTEGER RANGE)                                                                        │ -│[default: 2; 1<=x<=4]                                                                  │ -│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ -│[default: 3.7 3.8 3.9 3.10]                                                   │ -│--kubernetes-versionsKubernetes versions used to run in parallel (space separated).(TEXT)│ -│[default: v1.25.2 v1.24.6 v1.23.12 v1.22.15 v1.21.14]         │ -│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ -│--debug-resourcesWhether to show resource information while running in parallel.│ -│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze k8s tests [OPTIONS] [TEST_ARGS]... + +Run tests against the current KinD cluster (optionally for all clusters in parallel). + +╭─ K8S tests flags ────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.7< | 3.8 | 3.9 | 3.10 | 3.11)                           │ +│[default: 3.7]                                              │ +│--kubernetes-versionKubernetes version used to create the KinD cluster of.│ +│(>v1.25.2< | v1.24.6 | v1.23.12 | v1.22.15 | v1.21.14)│ +│[default: v1.25.2]                                    │ +│--executorExecutor to use for a kubernetes cluster.                                         │ +│(>KubernetesExecutor< | CeleryExecutor | LocalExecutor | CeleryKubernetesExecutor)│ +│[default: KubernetesExecutor]                                                     │ +│--force-venv-setupForce recreation of the virtualenv.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Parallel options ───────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ +│--parallelismMaximum number of processes to use while running the operation in parallel for cluster │ +│operations.                                                                            │ +│(INTEGER RANGE)                                                                        │ +│[default: 2; 1<=x<=4]                                                                  │ +│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ +│[default: 3.7 3.8 3.9 3.10 3.11]                                              │ +│--kubernetes-versionsKubernetes versions used to run in parallel (space separated).(TEXT)│ +│[default: v1.25.2 v1.24.6 v1.23.12 v1.22.15 v1.21.14]         │ +│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ +│--debug-resourcesWhether to show resource information while running in parallel.│ +│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_k8s_upload-k8s-image.svg b/images/breeze/output_k8s_upload-k8s-image.svg index c3310fee2b3b9..e92ced809075d 100644 --- a/images/breeze/output_k8s_upload-k8s-image.svg +++ b/images/breeze/output_k8s_upload-k8s-image.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - Command: k8s upload-k8s-image + Command: k8s upload-k8s-image - + - - -Usage: breeze k8s upload-k8s-image [OPTIONS] - -Upload k8s-ready airflow image to the KinD cluster (optionally to all clusters in parallel) - -╭─ Upload image flags ─────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--kubernetes-versionKubernetes version used to create the KinD cluster of.│ -│(>v1.25.2< | v1.24.6 | v1.23.12 | v1.22.15 | v1.21.14)│ -│[default: v1.25.2]                                    │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Parallel options ───────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ -│--parallelismMaximum number of processes to use while running the operation in parallel.│ -│(INTEGER RANGE)                                                            │ -│[default: 4; 1<=x<=8]                                                      │ -│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ -│[default: 3.7 3.8 3.9 3.10]                                                   │ -│--kubernetes-versionsKubernetes versions used to run in parallel (space separated).(TEXT)│ -│[default: v1.25.2 v1.24.6 v1.23.12 v1.22.15 v1.21.14]         │ -│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ -│--debug-resourcesWhether to show resource information while running in parallel.│ -│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze k8s upload-k8s-image [OPTIONS] + +Upload k8s-ready airflow image to the KinD cluster (optionally to all clusters in parallel) + +╭─ Upload image flags ─────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.7< | 3.8 | 3.9 | 3.10 | 3.11)                           │ +│[default: 3.7]                                              │ +│--kubernetes-versionKubernetes version used to create the KinD cluster of.│ +│(>v1.25.2< | v1.24.6 | v1.23.12 | v1.22.15 | v1.21.14)│ +│[default: v1.25.2]                                    │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Parallel options ───────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ +│--parallelismMaximum number of processes to use while running the operation in parallel.│ +│(INTEGER RANGE)                                                            │ +│[default: 4; 1<=x<=8]                                                      │ +│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ +│[default: 3.7 3.8 3.9 3.10 3.11]                                              │ +│--kubernetes-versionsKubernetes versions used to run in parallel (space separated).(TEXT)│ +│[default: v1.25.2 v1.24.6 v1.23.12 v1.22.15 v1.21.14]         │ +│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ +│--debug-resourcesWhether to show resource information while running in parallel.│ +│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_prod-image.svg b/images/breeze/output_prod-image.svg index 95374cf21d4b9..eb88437f63e2c 100644 --- a/images/breeze/output_prod-image.svg +++ b/images/breeze/output_prod-image.svg @@ -35,8 +35,8 @@ .terminal-3437848795-r1 { fill: #c5c8c6;font-weight: bold } .terminal-3437848795-r2 { fill: #c5c8c6 } .terminal-3437848795-r3 { fill: #d0b344;font-weight: bold } -.terminal-3437848795-r4 { fill: #868887 } -.terminal-3437848795-r5 { fill: #68a0b3;font-weight: bold } +.terminal-3437848795-r4 { fill: #68a0b3;font-weight: bold } +.terminal-3437848795-r5 { fill: #868887 } .terminal-3437848795-r6 { fill: #98a84b;font-weight: bold } @@ -93,18 +93,18 @@ -Usage: breeze prod-image [OPTIONS] COMMAND [ARGS]... +Usage: breeze prod-image [OPTIONS] COMMAND [ARGS]... -Tools that developers can use to manually manage PROD images +Tools that developers can use to manually manage PROD images -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Production Image tools ─────────────────────────────────────────────────────────────────────────────────────────────╮ -│build  Build Production image. Include building multiple images for all or selected Python versions sequentially. │ -│pull   Pull and optionally verify Production images - possibly in parallel for all Python versions.               │ -│verify Verify Production image.                                                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Production Image tools ─────────────────────────────────────────────────────────────────────────────────────────────╮ +│build  Build Production image. Include building multiple images for all or selected Python versions sequentially. │ +│pull   Pull and optionally verify Production images - possibly in parallel for all Python versions.               │ +│verify Verify Production image.                                                                                   │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_prod-image_build.svg b/images/breeze/output_prod-image_build.svg index e199832081ba2..19d4dd68ee2ca 100644 --- a/images/breeze/output_prod-image_build.svg +++ b/images/breeze/output_prod-image_build.svg @@ -19,393 +19,393 @@ font-weight: 700; } - .terminal-3548632731-matrix { + .terminal-2387007357-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-3548632731-title { + .terminal-2387007357-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-3548632731-r1 { fill: #c5c8c6;font-weight: bold } -.terminal-3548632731-r2 { fill: #c5c8c6 } -.terminal-3548632731-r3 { fill: #d0b344;font-weight: bold } -.terminal-3548632731-r4 { fill: #868887 } -.terminal-3548632731-r5 { fill: #68a0b3;font-weight: bold } -.terminal-3548632731-r6 { fill: #98a84b;font-weight: bold } -.terminal-3548632731-r7 { fill: #8d7b39 } + .terminal-2387007357-r1 { fill: #c5c8c6;font-weight: bold } +.terminal-2387007357-r2 { fill: #c5c8c6 } +.terminal-2387007357-r3 { fill: #d0b344;font-weight: bold } +.terminal-2387007357-r4 { fill: #68a0b3;font-weight: bold } +.terminal-2387007357-r5 { fill: #868887 } +.terminal-2387007357-r6 { fill: #98a84b;font-weight: bold } +.terminal-2387007357-r7 { fill: #8d7b39 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - Command: prod-image build + Command: prod-image build - + - - -Usage: breeze prod-image build [OPTIONS] - -Build Production image. Include building multiple images for all or selected Python versions sequentially. - -╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.│ -│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ -│[default: 3.7]                                              │ -│--install-airflow-version-VInstall version of Airflow from PyPI.(TEXT)│ -│--upgrade-to-newer-dependencies-uWhen set, upgrade all PIP packages to latest.│ -│--upgrade-on-failureWhen set, attempt to run upgrade to newer dependencies when regular build      │ -│fails.                                                                         │ -│--image-tag-tTag the image after building it.(TEXT)[default: latest]│ -│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful│ -│when you build or pull image with --image-tag.                                 │ -│--docker-cache-cCache option for image used during the build.(registry | local | disabled)│ -│[default: registry]                          │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Building images in parallel ────────────────────────────────────────────────────────────────────────────────────────╮ -│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ -│--parallelismMaximum number of processes to use while running the operation in parallel.│ -│(INTEGER RANGE)                                                            │ -│[default: 4; 1<=x<=8]                                                      │ -│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ -│[default: 3.7 3.8 3.9 3.10]                                                   │ -│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ -│--debug-resourcesWhether to show resource information while running in parallel.│ -│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options for customizing images ─────────────────────────────────────────────────────────────────────────────────────╮ -│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)│ -│--install-providers-from-sourcesInstall providers from sources when installing.│ -│--airflow-extrasExtras to install by default.                                                   │ -│(TEXT)                                                                          │ -│[default:                                                                       │ -│amazon,async,celery,cncf.kubernetes,dask,docker,elasticsearch,ftp,google,google…│ -│--airflow-constraints-modeMode of constraints for PROD image building.                           │ -│(constraints | constraints-no-providers | constraints-source-providers)│ -│[default: constraints]                                                 │ -│--airflow-constraints-referenceConstraint reference to use when building the image.(TEXT)│ -│--python-imageIf specified this is the base python image used to build the image. Should be   │ -│something like: python:VERSION-slim-bullseye.                                   │ -│(TEXT)                                                                          │ -│--additional-extrasAdditional extra package while installing Airflow in the image.(TEXT)│ -│--additional-pip-install-flagsAdditional flags added to `pip install` commands (except reinstalling `pip`     │ -│itself).                                                                        │ -│(TEXT)                                                                          │ -│--additional-python-depsAdditional python dependencies to use when building the images.(TEXT)│ -│--additional-runtime-apt-depsAdditional apt runtime dependencies to use when building the images.(TEXT)│ -│--additional-runtime-apt-envAdditional environment variables set when adding runtime dependencies.(TEXT)│ -│--additional-runtime-apt-commandAdditional command executed before runtime apt deps are installed.(TEXT)│ -│--additional-dev-apt-depsAdditional apt dev dependencies to use when building the images.(TEXT)│ -│--additional-dev-apt-envAdditional environment variables set when adding dev dependencies.(TEXT)│ -│--additional-dev-apt-commandAdditional command executed before dev apt deps are installed.(TEXT)│ -│--runtime-apt-depsApt runtime dependencies to use when building the images.(TEXT)│ -│--runtime-apt-commandCommand executed before runtime apt deps are installed.(TEXT)│ -│--dev-apt-depsApt dev dependencies to use when building the images.(TEXT)│ -│--dev-apt-commandCommand executed before dev apt deps are installed.(TEXT)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Customization options (for specific customization needs) ───────────────────────────────────────────────────────────╮ -│--install-packages-from-contextInstall wheels from local docker-context-files when building image.       │ -│Implies --disable-airflow-repo-cache.                                     │ -│--cleanup-contextClean up docker context files before running build (cannot be used        │ -│together with --install-packages-from-context).                           │ -│--disable-mysql-client-installationDo not install MySQL client.│ -│--disable-mssql-client-installationDo not install MsSQl client.│ -│--disable-postgres-client-installationDo not install Postgres client.│ -│--disable-airflow-repo-cacheDisable cache from Airflow repository during building.│ -│--install-airflow-referenceInstall Airflow using GitHub tag or branch.(TEXT)│ -│--installation-methodInstall Airflow from: sources or PyPI.(. | apache-airflow)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ -│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ -│--github-usernameThe user name used to authenticate to GitHub.(TEXT)│ -│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)│ -│--pushPush image after building it.│ -│--empty-imagePrepare empty image tagged with the same name as the Airflow image.│ -│--prepare-buildx-cachePrepares build cache (this is done as separate per-platform steps instead of building the │ -│image).                                                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--verbose-vPrint verbose information about performed steps.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze prod-image build [OPTIONS] + +Build Production image. Include building multiple images for all or selected Python versions sequentially. + +╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.7< | 3.8 | 3.9 | 3.10 | 3.11)                           │ +│[default: 3.7]                                              │ +│--install-airflow-version-VInstall version of Airflow from PyPI.(TEXT)│ +│--upgrade-to-newer-dependencies-uWhen set, upgrade all PIP packages to latest.│ +│--upgrade-on-failureWhen set, attempt to run upgrade to newer dependencies when regular build      │ +│fails.                                                                         │ +│--image-tag-tTag the image after building it.(TEXT)[default: latest]│ +│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful│ +│when you build or pull image with --image-tag.                                 │ +│--docker-cache-cCache option for image used during the build.(registry | local | disabled)│ +│[default: registry]                          │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Building images in parallel ────────────────────────────────────────────────────────────────────────────────────────╮ +│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ +│--parallelismMaximum number of processes to use while running the operation in parallel.│ +│(INTEGER RANGE)                                                            │ +│[default: 4; 1<=x<=8]                                                      │ +│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ +│[default: 3.7 3.8 3.9 3.10 3.11]                                              │ +│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ +│--debug-resourcesWhether to show resource information while running in parallel.│ +│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options for customizing images ─────────────────────────────────────────────────────────────────────────────────────╮ +│--builderBuildx builder used to perform `docker buildx build` commands.(TEXT)│ +│--install-providers-from-sourcesInstall providers from sources when installing.│ +│--airflow-extrasExtras to install by default.                                                   │ +│(TEXT)                                                                          │ +│[default:                                                                       │ +│amazon,async,celery,cncf.kubernetes,dask,docker,elasticsearch,ftp,google,google…│ +│--airflow-constraints-modeMode of constraints for PROD image building.                           │ +│(constraints | constraints-no-providers | constraints-source-providers)│ +│[default: constraints]                                                 │ +│--airflow-constraints-referenceConstraint reference to use when building the image.(TEXT)│ +│--python-imageIf specified this is the base python image used to build the image. Should be   │ +│something like: python:VERSION-slim-bullseye.                                   │ +│(TEXT)                                                                          │ +│--additional-extrasAdditional extra package while installing Airflow in the image.(TEXT)│ +│--additional-pip-install-flagsAdditional flags added to `pip install` commands (except reinstalling `pip`     │ +│itself).                                                                        │ +│(TEXT)                                                                          │ +│--additional-python-depsAdditional python dependencies to use when building the images.(TEXT)│ +│--additional-runtime-apt-depsAdditional apt runtime dependencies to use when building the images.(TEXT)│ +│--additional-runtime-apt-envAdditional environment variables set when adding runtime dependencies.(TEXT)│ +│--additional-runtime-apt-commandAdditional command executed before runtime apt deps are installed.(TEXT)│ +│--additional-dev-apt-depsAdditional apt dev dependencies to use when building the images.(TEXT)│ +│--additional-dev-apt-envAdditional environment variables set when adding dev dependencies.(TEXT)│ +│--additional-dev-apt-commandAdditional command executed before dev apt deps are installed.(TEXT)│ +│--runtime-apt-depsApt runtime dependencies to use when building the images.(TEXT)│ +│--runtime-apt-commandCommand executed before runtime apt deps are installed.(TEXT)│ +│--dev-apt-depsApt dev dependencies to use when building the images.(TEXT)│ +│--dev-apt-commandCommand executed before dev apt deps are installed.(TEXT)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Customization options (for specific customization needs) ───────────────────────────────────────────────────────────╮ +│--install-packages-from-contextInstall wheels from local docker-context-files when building image.       │ +│Implies --disable-airflow-repo-cache.                                     │ +│--cleanup-contextClean up docker context files before running build (cannot be used        │ +│together with --install-packages-from-context).                           │ +│--disable-mysql-client-installationDo not install MySQL client.│ +│--disable-mssql-client-installationDo not install MsSQl client.│ +│--disable-postgres-client-installationDo not install Postgres client.│ +│--disable-airflow-repo-cacheDisable cache from Airflow repository during building.│ +│--install-airflow-referenceInstall Airflow using GitHub tag or branch.(TEXT)│ +│--installation-methodInstall Airflow from: sources or PyPI.(. | apache-airflow)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ +│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ +│--github-usernameThe user name used to authenticate to GitHub.(TEXT)│ +│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)│ +│--pushPush image after building it.│ +│--empty-imagePrepare empty image tagged with the same name as the Airflow image.│ +│--prepare-buildx-cachePrepares build cache (this is done as separate per-platform steps instead of building the │ +│image).                                                                                   │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--verbose-vPrint verbose information about performed steps.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_prod-image_pull.svg b/images/breeze/output_prod-image_pull.svg index 460793f165c54..67a0d14560169 100644 --- a/images/breeze/output_prod-image_pull.svg +++ b/images/breeze/output_prod-image_pull.svg @@ -19,169 +19,169 @@ font-weight: 700; } - .terminal-930179811-matrix { + .terminal-1427270597-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-930179811-title { + .terminal-1427270597-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-930179811-r1 { fill: #c5c8c6;font-weight: bold } -.terminal-930179811-r2 { fill: #c5c8c6 } -.terminal-930179811-r3 { fill: #d0b344;font-weight: bold } -.terminal-930179811-r4 { fill: #868887 } -.terminal-930179811-r5 { fill: #68a0b3;font-weight: bold } -.terminal-930179811-r6 { fill: #98a84b;font-weight: bold } -.terminal-930179811-r7 { fill: #8d7b39 } + .terminal-1427270597-r1 { fill: #c5c8c6;font-weight: bold } +.terminal-1427270597-r2 { fill: #c5c8c6 } +.terminal-1427270597-r3 { fill: #d0b344;font-weight: bold } +.terminal-1427270597-r4 { fill: #68a0b3;font-weight: bold } +.terminal-1427270597-r5 { fill: #868887 } +.terminal-1427270597-r6 { fill: #98a84b;font-weight: bold } +.terminal-1427270597-r7 { fill: #8d7b39 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - Command: prod-image pull + Command: prod-image pull - + - - -Usage: breeze prod-image pull [OPTIONS] [EXTRA_PYTEST_ARGS]... - -Pull and optionally verify Production images - possibly in parallel for all Python versions. - -╭─ Pull image flags ───────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--image-tag-tTag of the image which is used to pull the image.(TEXT)[default: latest]│ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ -│--verifyVerify image.│ -│--wait-for-imageWait until image is available.│ -│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful when you build│ -│or pull image with --image-tag.                                                               │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Parallel running ───────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ -│--parallelismMaximum number of processes to use while running the operation in parallel.│ -│(INTEGER RANGE)                                                            │ -│[default: 4; 1<=x<=8]                                                      │ -│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ -│[default: 3.7 3.8 3.9 3.10]                                                   │ -│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ -│--debug-resourcesWhether to show resource information while running in parallel.│ -│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze prod-image pull [OPTIONS] [EXTRA_PYTEST_ARGS]... + +Pull and optionally verify Production images - possibly in parallel for all Python versions. + +╭─ Pull image flags ───────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--image-tag-tTag of the image which is used to pull the image.(TEXT)[default: latest]│ +│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10 | 3.11)│ +│[default: 3.7]                                              │ +│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ +│--verifyVerify image.│ +│--wait-for-imageWait until image is available.│ +│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful when you build│ +│or pull image with --image-tag.                                                               │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Parallel running ───────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ +│--parallelismMaximum number of processes to use while running the operation in parallel.│ +│(INTEGER RANGE)                                                            │ +│[default: 4; 1<=x<=8]                                                      │ +│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ +│[default: 3.7 3.8 3.9 3.10 3.11]                                              │ +│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ +│--debug-resourcesWhether to show resource information while running in parallel.│ +│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_prod-image_verify.svg b/images/breeze/output_prod-image_verify.svg index e193df088984b..57317cec86172 100644 --- a/images/breeze/output_prod-image_verify.svg +++ b/images/breeze/output_prod-image_verify.svg @@ -19,117 +19,117 @@ font-weight: 700; } - .terminal-591802771-matrix { + .terminal-912011826-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-591802771-title { + .terminal-912011826-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-591802771-r1 { fill: #c5c8c6;font-weight: bold } -.terminal-591802771-r2 { fill: #c5c8c6 } -.terminal-591802771-r3 { fill: #d0b344;font-weight: bold } -.terminal-591802771-r4 { fill: #868887 } -.terminal-591802771-r5 { fill: #68a0b3;font-weight: bold } -.terminal-591802771-r6 { fill: #98a84b;font-weight: bold } -.terminal-591802771-r7 { fill: #8d7b39 } + .terminal-912011826-r1 { fill: #c5c8c6;font-weight: bold } +.terminal-912011826-r2 { fill: #c5c8c6 } +.terminal-912011826-r3 { fill: #d0b344;font-weight: bold } +.terminal-912011826-r4 { fill: #68a0b3;font-weight: bold } +.terminal-912011826-r5 { fill: #868887 } +.terminal-912011826-r6 { fill: #98a84b;font-weight: bold } +.terminal-912011826-r7 { fill: #8d7b39 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - Command: prod-image verify + Command: prod-image verify - + - - -Usage: breeze prod-image verify [OPTIONS] [EXTRA_PYTEST_ARGS]... - -Verify Production image. - -╭─ Verify image flags ─────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--image-name-nName of the image to verify (overrides --python and --image-tag).(TEXT)│ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--slim-imageThe image to verify is slim and non-slim tests should be skipped.│ -│--image-tag-tTag of the image when verifying it.(TEXT)[default: latest]│ -│--pullPull image is missing before attempting to verify it.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze prod-image verify [OPTIONS] [EXTRA_PYTEST_ARGS]... + +Verify Production image. + +╭─ Verify image flags ─────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--image-name-nName of the image to verify (overrides --python and --image-tag).(TEXT)│ +│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10 | 3.11)│ +│[default: 3.7]                                              │ +│--slim-imageThe image to verify is slim and non-slim tests should be skipped.│ +│--image-tag-tTag of the image when verifying it.(TEXT)[default: latest]│ +│--pullPull image is missing before attempting to verify it.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_release-management.svg b/images/breeze/output_release-management.svg index 66af753517d41..87988392239d0 100644 --- a/images/breeze/output_release-management.svg +++ b/images/breeze/output_release-management.svg @@ -35,8 +35,8 @@ .terminal-3090088729-r1 { fill: #c5c8c6;font-weight: bold } .terminal-3090088729-r2 { fill: #c5c8c6 } .terminal-3090088729-r3 { fill: #d0b344;font-weight: bold } -.terminal-3090088729-r4 { fill: #868887 } -.terminal-3090088729-r5 { fill: #68a0b3;font-weight: bold } +.terminal-3090088729-r4 { fill: #68a0b3;font-weight: bold } +.terminal-3090088729-r5 { fill: #868887 } .terminal-3090088729-r6 { fill: #98a84b;font-weight: bold } @@ -102,21 +102,21 @@ -Usage: breeze release-management [OPTIONS] COMMAND [ARGS]... +Usage: breeze release-management [OPTIONS] COMMAND [ARGS]... Tools that release managers can use to prepare and manage Airflow releases -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Commands ───────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│generate-constraints             Generates pinned constraint files with all extras from setup.py in parallel.     │ -│prepare-airflow-package          Prepare sdist/whl package of Airflow.                                            │ -│prepare-provider-documentation   Prepare CHANGELOG, README and COMMITS information for providers.                 │ -│prepare-provider-packages        Prepare sdist/whl packages of Airflow Providers.                                 │ -│release-prod-images              Release production images to DockerHub (needs DockerHub permissions).            │ -│verify-provider-packages         Verifies if all provider code is following expectations for providers.           │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Commands ───────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│generate-constraints             Generates pinned constraint files with all extras from setup.py in parallel.     │ +│prepare-airflow-package          Prepare sdist/whl package of Airflow.                                            │ +│prepare-provider-documentation   Prepare CHANGELOG, README and COMMITS information for providers.                 │ +│prepare-provider-packages        Prepare sdist/whl packages of Airflow Providers.                                 │ +│release-prod-images              Release production images to DockerHub (needs DockerHub permissions).            │ +│verify-provider-packages         Verifies if all provider code is following expectations for providers.           │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_release-management_generate-constraints.svg b/images/breeze/output_release-management_generate-constraints.svg index 5ec7bda74e351..9b19194060440 100644 --- a/images/breeze/output_release-management_generate-constraints.svg +++ b/images/breeze/output_release-management_generate-constraints.svg @@ -19,173 +19,173 @@ font-weight: 700; } - .terminal-1117746507-matrix { + .terminal-4225791533-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1117746507-title { + .terminal-4225791533-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1117746507-r1 { fill: #c5c8c6;font-weight: bold } -.terminal-1117746507-r2 { fill: #c5c8c6 } -.terminal-1117746507-r3 { fill: #d0b344;font-weight: bold } -.terminal-1117746507-r4 { fill: #868887 } -.terminal-1117746507-r5 { fill: #68a0b3;font-weight: bold } -.terminal-1117746507-r6 { fill: #98a84b;font-weight: bold } -.terminal-1117746507-r7 { fill: #8d7b39 } + .terminal-4225791533-r1 { fill: #c5c8c6;font-weight: bold } +.terminal-4225791533-r2 { fill: #c5c8c6 } +.terminal-4225791533-r3 { fill: #d0b344;font-weight: bold } +.terminal-4225791533-r4 { fill: #68a0b3;font-weight: bold } +.terminal-4225791533-r5 { fill: #868887 } +.terminal-4225791533-r6 { fill: #98a84b;font-weight: bold } +.terminal-4225791533-r7 { fill: #8d7b39 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - Command: release-management generate-constraints + Command: release-management generate-constraints - + - - -Usage: breeze release-management generate-constraints [OPTIONS] - -Generates pinned constraint files with all extras from setup.py in parallel. - -╭─ Generate constraints flags ─────────────────────────────────────────────────────────────────────────────────────────╮ -│--image-tag-tTag of the image which is used to run the image (implies --mount-sources=skip).│ -│(TEXT)                                                                         │ -│[default: latest]                                                              │ -│--python-pPython major/minor version used in Airflow image for images.│ -│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ -│[default: 3.7]                                              │ -│--airflow-constraints-modeMode of constraints for CI image building.                             │ -│(constraints-source-providers | constraints | constraints-no-providers)│ -│[default: constraints-source-providers]                                │ -│--debugDrop user in shell instead of running the command. Useful for debugging.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Parallel running ───────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ -│--parallelismMaximum number of processes to use while running the operation in parallel.(INTEGER RANGE)│ -│[default: 4; 1<=x<=8]                                                      │ -│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ -│[default: 3.7 3.8 3.9 3.10]                                                   │ -│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ -│--debug-resourcesWhether to show resource information while running in parallel.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze release-management generate-constraints [OPTIONS] + +Generates pinned constraint files with all extras from setup.py in parallel. + +╭─ Generate constraints flags ─────────────────────────────────────────────────────────────────────────────────────────╮ +│--image-tag-tTag of the image which is used to run the image (implies --mount-sources=skip).│ +│(TEXT)                                                                         │ +│[default: latest]                                                              │ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.7< | 3.8 | 3.9 | 3.10 | 3.11)                           │ +│[default: 3.7]                                              │ +│--airflow-constraints-modeMode of constraints for CI image building.                             │ +│(constraints-source-providers | constraints | constraints-no-providers)│ +│[default: constraints-source-providers]                                │ +│--debugDrop user in shell instead of running the command. Useful for debugging.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Parallel running ───────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ +│--parallelismMaximum number of processes to use while running the operation in parallel.(INTEGER RANGE)│ +│[default: 4; 1<=x<=8]                                                      │ +│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ +│[default: 3.7 3.8 3.9 3.10 3.11]                                              │ +│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ +│--debug-resourcesWhether to show resource information while running in parallel.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_release-management_release-prod-images.svg b/images/breeze/output_release-management_release-prod-images.svg index ef945e9fd0ed2..e29fed1f7c18e 100644 --- a/images/breeze/output_release-management_release-prod-images.svg +++ b/images/breeze/output_release-management_release-prod-images.svg @@ -19,139 +19,139 @@ font-weight: 700; } - .terminal-2418532981-matrix { + .terminal-560063252-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2418532981-title { + .terminal-560063252-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2418532981-r1 { fill: #c5c8c6;font-weight: bold } -.terminal-2418532981-r2 { fill: #c5c8c6 } -.terminal-2418532981-r3 { fill: #d0b344;font-weight: bold } -.terminal-2418532981-r4 { fill: #868887 } -.terminal-2418532981-r5 { fill: #cc555a } -.terminal-2418532981-r6 { fill: #68a0b3;font-weight: bold } -.terminal-2418532981-r7 { fill: #8d7b39 } -.terminal-2418532981-r8 { fill: #8a4346 } -.terminal-2418532981-r9 { fill: #98a84b;font-weight: bold } + .terminal-560063252-r1 { fill: #c5c8c6;font-weight: bold } +.terminal-560063252-r2 { fill: #c5c8c6 } +.terminal-560063252-r3 { fill: #d0b344;font-weight: bold } +.terminal-560063252-r4 { fill: #68a0b3;font-weight: bold } +.terminal-560063252-r5 { fill: #868887 } +.terminal-560063252-r6 { fill: #cc555a } +.terminal-560063252-r7 { fill: #8d7b39 } +.terminal-560063252-r8 { fill: #8a4346 } +.terminal-560063252-r9 { fill: #98a84b;font-weight: bold } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - Command: release-management release-prod-images + Command: release-management release-prod-images - + - - -Usage: breeze release-management release-prod-images [OPTIONS] - -Release production images to DockerHub (needs DockerHub permissions). - -╭─ Release PROD IMAGE flags ───────────────────────────────────────────────────────────────────────────────────────────╮ -│*--airflow-versionAirflow version to release (2.3.0, 2.3.0rc1 etc.)(TEXT)[required]│ -│--dockerhub-repoDockerHub repository for the images(TEXT)[default: apache/airflow]│ -│--slim-imagesWhether to prepare slim images instead of the regular ones.│ -│--limit-pythonSpecific python to build slim images for (if not specified - the images are built for all   │ -│available python versions)                                                                  │ -│(3.7 | 3.8 | 3.9 | 3.10)                                                                    │ -│--limit-platformSpecific platform to build images for (if not specified, multiplatform images will be built.│ -│(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)                                       │ -│[default: linux/amd64,linux/arm64]                                                          │ -│--skip-latestWhether to skip publishing the latest images (so that 'latest' images are not updated). This│ -│should only be used if you release image for previous branches. Automatically set when      │ -│rc/alpha/beta images are built.                                                             │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze release-management release-prod-images [OPTIONS] + +Release production images to DockerHub (needs DockerHub permissions). + +╭─ Release PROD IMAGE flags ───────────────────────────────────────────────────────────────────────────────────────────╮ +│*--airflow-versionAirflow version to release (2.3.0, 2.3.0rc1 etc.)(TEXT)[required]│ +│--dockerhub-repoDockerHub repository for the images(TEXT)[default: apache/airflow]│ +│--slim-imagesWhether to prepare slim images instead of the regular ones.│ +│--limit-pythonSpecific python to build slim images for (if not specified - the images are built for all   │ +│available python versions)                                                                  │ +│(3.7 | 3.8 | 3.9 | 3.10 | 3.11)                                                             │ +│--limit-platformSpecific platform to build images for (if not specified, multiplatform images will be built.│ +│(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)                                       │ +│[default: linux/amd64,linux/arm64]                                                          │ +│--skip-latestWhether to skip publishing the latest images (so that 'latest' images are not updated). This│ +│should only be used if you release image for previous branches. Automatically set when      │ +│rc/alpha/beta images are built.                                                             │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_setup.svg b/images/breeze/output_setup.svg index 40050418b0348..4ea9da3ed8a9e 100644 --- a/images/breeze/output_setup.svg +++ b/images/breeze/output_setup.svg @@ -35,8 +35,8 @@ .terminal-563718598-r1 { fill: #c5c8c6;font-weight: bold } .terminal-563718598-r2 { fill: #c5c8c6 } .terminal-563718598-r3 { fill: #d0b344;font-weight: bold } -.terminal-563718598-r4 { fill: #868887 } -.terminal-563718598-r5 { fill: #68a0b3;font-weight: bold } +.terminal-563718598-r4 { fill: #68a0b3;font-weight: bold } +.terminal-563718598-r5 { fill: #868887 } .terminal-563718598-r6 { fill: #98a84b;font-weight: bold } @@ -99,20 +99,20 @@ -Usage: breeze setup [OPTIONS] COMMAND [ARGS]... +Usage: breeze setup [OPTIONS] COMMAND [ARGS]... Tools that developers can use to configure Breeze -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Commands ───────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│autocomplete                    Enables autocompletion of breeze commands.                                        │ -│config                          Show/update configuration (Python, Backend, Cheatsheet, ASCIIART).                │ -│regenerate-command-images       Regenerate breeze command images.                                                 │ -│self-upgrade                    Self upgrade Breeze.                                                              │ -│version                         Print information about version of apache-airflow-breeze.                         │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Commands ───────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│autocomplete                    Enables autocompletion of breeze commands.                                        │ +│config                          Show/update configuration (Python, Backend, Cheatsheet, ASCIIART).                │ +│regenerate-command-images       Regenerate breeze command images.                                                 │ +│self-upgrade                    Self upgrade Breeze.                                                              │ +│version                         Print information about version of apache-airflow-breeze.                         │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_setup_config.svg b/images/breeze/output_setup_config.svg index 4d6c2dbba5af0..89bc2fd5ee698 100644 --- a/images/breeze/output_setup_config.svg +++ b/images/breeze/output_setup_config.svg @@ -19,125 +19,125 @@ font-weight: 700; } - .terminal-4106968381-matrix { + .terminal-3123797468-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-4106968381-title { + .terminal-3123797468-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-4106968381-r1 { fill: #c5c8c6;font-weight: bold } -.terminal-4106968381-r2 { fill: #c5c8c6 } -.terminal-4106968381-r3 { fill: #d0b344;font-weight: bold } -.terminal-4106968381-r4 { fill: #868887 } -.terminal-4106968381-r5 { fill: #68a0b3;font-weight: bold } -.terminal-4106968381-r6 { fill: #98a84b;font-weight: bold } -.terminal-4106968381-r7 { fill: #8d7b39 } + .terminal-3123797468-r1 { fill: #c5c8c6;font-weight: bold } +.terminal-3123797468-r2 { fill: #c5c8c6 } +.terminal-3123797468-r3 { fill: #d0b344;font-weight: bold } +.terminal-3123797468-r4 { fill: #68a0b3;font-weight: bold } +.terminal-3123797468-r5 { fill: #868887 } +.terminal-3123797468-r6 { fill: #98a84b;font-weight: bold } +.terminal-3123797468-r7 { fill: #8d7b39 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - Command: setup config + Command: setup config - + - - -Usage: breeze setup config [OPTIONS] - -Show/update configuration (Python, Backend, Cheatsheet, ASCIIART). - -╭─ Config flags ───────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.│ -│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ -│[default: 3.7]                                              │ -│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)│ -│[default: sqlite]       │ -│--postgres-version-PVersion of Postgres used.(>10< | 11 | 12 | 13 | 14)[default: 10]│ -│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ -│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ -│--cheatsheet/--no-cheatsheet-C/-cEnable/disable cheatsheet.│ -│--asciiart/--no-asciiart-A/-aEnable/disable ASCIIart.│ -│--colour/--no-colourEnable/disable Colour mode (useful for colour blind-friendly communication).│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze setup config [OPTIONS] + +Show/update configuration (Python, Backend, Cheatsheet, ASCIIART). + +╭─ Config flags ───────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.7< | 3.8 | 3.9 | 3.10 | 3.11)                           │ +│[default: 3.7]                                              │ +│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)│ +│[default: sqlite]       │ +│--postgres-version-PVersion of Postgres used.(>10< | 11 | 12 | 13 | 14)[default: 10]│ +│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ +│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ +│--cheatsheet/--no-cheatsheet-C/-cEnable/disable cheatsheet.│ +│--asciiart/--no-asciiart-A/-aEnable/disable ASCIIart.│ +│--colour/--no-colourEnable/disable Colour mode (useful for colour blind-friendly communication).│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_shell.svg b/images/breeze/output_shell.svg index def029f80b4f6..df6d4814c932a 100644 --- a/images/breeze/output_shell.svg +++ b/images/breeze/output_shell.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - Command: shell + Command: shell - + - - -Usage: breeze shell [OPTIONS] [EXTRA_ARGS]... - -Enter breeze environment. this is the default command use when no other is selected. - -╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ -│--postgres-version-PVersion of Postgres used.(>10< | 11 | 12 | 13 | 14)[default: 10]│ -│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ -│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ -│--integrationIntegration(s) to enable when running (can be more than one).                            │ -│(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino |   │ -│all)                                                                                     │ -│--forward-credentials-fForward local credentials to container when running.│ -│--db-reset-dReset DB when entering the container.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced flag for running ──────────────────────────────────────────────────────────────────────────────────────────╮ -│--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It can also be `none`,     │ -│`wheel`, or `sdist` if Airflow should be removed, installed from wheel packages│ -│or sdist packages available in dist folder respectively. Implies               │ -│--mount-sources `remove`.                                                      │ -│(none | wheel | sdist | <airflow_version>)                                     │ -│--airflow-constraints-referenceConstraint reference to use. Useful with --use-airflow-version parameter to    │ -│specify constraints for the installed version and to find newer dependencies   │ -│(TEXT)                                                                         │ -│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64)│ -│--airflow-extrasAirflow extras to install when --use-airflow-version is used(TEXT)│ -│--use-packages-from-distInstall all found packages (--package-format determines type) from 'dist'      │ -│folder when entering breeze.                                                   │ -│--package-formatFormat of packages that should be installed from dist.(wheel | sdist)│ -│[default: wheel]                                      │ -│--force-buildForce image build no matter if it is determined as needed.│ -│--image-tag-tTag of the image which is used to run the image (implies --mount-sources=skip).│ -│(TEXT)                                                                         │ -│[default: latest]                                                              │ -│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed      │ -│(default = selected).                                                          │ -│(selected | all | skip | remove)                                               │ -│[default: selected]                                                            │ -│--include-mypy-volumeWhether to include mounting of the mypy volume (useful for debugging mypy).│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--max-timeMaximum time that the command should take - if it takes longer, the command will fail.│ -│(INTEGER RANGE)                                                                       │ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze shell [OPTIONS] [EXTRA_ARGS]... + +Enter breeze environment. this is the default command use when no other is selected. + +╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.7< | 3.8 | 3.9 | 3.10 | 3.11)                           │ +│[default: 3.7]                                              │ +│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ +│--postgres-version-PVersion of Postgres used.(>10< | 11 | 12 | 13 | 14)[default: 10]│ +│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ +│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ +│--integrationIntegration(s) to enable when running (can be more than one).                            │ +│(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino |   │ +│all)                                                                                     │ +│--forward-credentials-fForward local credentials to container when running.│ +│--db-reset-dReset DB when entering the container.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced flag for running ──────────────────────────────────────────────────────────────────────────────────────────╮ +│--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It can also be `none`,     │ +│`wheel`, or `sdist` if Airflow should be removed, installed from wheel packages│ +│or sdist packages available in dist folder respectively. Implies               │ +│--mount-sources `remove`.                                                      │ +│(none | wheel | sdist | <airflow_version>)                                     │ +│--airflow-constraints-referenceConstraint reference to use. Useful with --use-airflow-version parameter to    │ +│specify constraints for the installed version and to find newer dependencies   │ +│(TEXT)                                                                         │ +│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64)│ +│--airflow-extrasAirflow extras to install when --use-airflow-version is used(TEXT)│ +│--use-packages-from-distInstall all found packages (--package-format determines type) from 'dist'      │ +│folder when entering breeze.                                                   │ +│--package-formatFormat of packages that should be installed from dist.(wheel | sdist)│ +│[default: wheel]                                      │ +│--force-buildForce image build no matter if it is determined as needed.│ +│--image-tag-tTag of the image which is used to run the image (implies --mount-sources=skip).│ +│(TEXT)                                                                         │ +│[default: latest]                                                              │ +│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed      │ +│(default = selected).                                                          │ +│(selected | all | skip | remove)                                               │ +│[default: selected]                                                            │ +│--include-mypy-volumeWhether to include mounting of the mypy volume (useful for debugging mypy).│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--max-timeMaximum time that the command should take - if it takes longer, the command will fail.│ +│(INTEGER RANGE)                                                                       │ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_start-airflow.svg b/images/breeze/output_start-airflow.svg index 82317dd6ada45..b344ca1f1b646 100644 --- a/images/breeze/output_start-airflow.svg +++ b/images/breeze/output_start-airflow.svg @@ -19,277 +19,277 @@ font-weight: 700; } - .terminal-2829072632-matrix { + .terminal-2991470999-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-2829072632-title { + .terminal-2991470999-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-2829072632-r1 { fill: #c5c8c6;font-weight: bold } -.terminal-2829072632-r2 { fill: #c5c8c6 } -.terminal-2829072632-r3 { fill: #d0b344;font-weight: bold } -.terminal-2829072632-r4 { fill: #868887 } -.terminal-2829072632-r5 { fill: #68a0b3;font-weight: bold } -.terminal-2829072632-r6 { fill: #98a84b;font-weight: bold } -.terminal-2829072632-r7 { fill: #8d7b39 } + .terminal-2991470999-r1 { fill: #c5c8c6;font-weight: bold } +.terminal-2991470999-r2 { fill: #c5c8c6 } +.terminal-2991470999-r3 { fill: #d0b344;font-weight: bold } +.terminal-2991470999-r4 { fill: #68a0b3;font-weight: bold } +.terminal-2991470999-r5 { fill: #868887 } +.terminal-2991470999-r6 { fill: #98a84b;font-weight: bold } +.terminal-2991470999-r7 { fill: #8d7b39 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - Command: start-airflow + Command: start-airflow - + - - -Usage: breeze start-airflow [OPTIONS] [EXTRA_ARGS]... - -Enter breeze environment and starts all Airflow components in the tmux session. Compile assets if contents of www  -directory changed. - -╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.│ -│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ -│[default: 3.7]                                              │ -│--load-example-dags-eEnable configuration to load example DAGs when starting Airflow.│ -│--load-default-connections-cEnable configuration to load default connections when starting Airflow.│ -│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ -│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64)│ -│--postgres-version-PVersion of Postgres used.(>10< | 11 | 12 | 13 | 14)[default: 10]│ -│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ -│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ -│--integrationIntegration(s) to enable when running (can be more than one).                       │ -│(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino│ -│| all)                                                                              │ -│--forward-credentials-fForward local credentials to container when running.│ -│--db-reset-dReset DB when entering the container.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Asset compilation options ──────────────────────────────────────────────────────────────────────────────────────────╮ -│--skip-asset-compilationSkips compilation of assets when starting airflow even if the content of www changed    │ -│(mutually exclusive with --dev-mode).                                                   │ -│--dev-modeStarts webserver in dev mode (assets are always recompiled in this case when starting)  │ -│(mutually exclusive with --skip-asset-compilation).                                     │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced flag for running ──────────────────────────────────────────────────────────────────────────────────────────╮ -│--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It can also be `none`,     │ -│`wheel`, or `sdist` if Airflow should be removed, installed from wheel packages│ -│or sdist packages available in dist folder respectively. Implies               │ -│--mount-sources `remove`.                                                      │ -│(none | wheel | sdist | <airflow_version>)                                     │ -│--airflow-constraints-referenceConstraint reference to use. Useful with --use-airflow-version parameter to    │ -│specify constraints for the installed version and to find newer dependencies   │ -│(TEXT)                                                                         │ -│--airflow-extrasAirflow extras to install when --use-airflow-version is used(TEXT)│ -│--use-packages-from-distInstall all found packages (--package-format determines type) from 'dist'      │ -│folder when entering breeze.                                                   │ -│--package-formatFormat of packages that should be installed from dist.(wheel | sdist)│ -│[default: wheel]                                      │ -│--force-buildForce image build no matter if it is determined as needed.│ -│--image-tag-tTag of the image which is used to run the image (implies --mount-sources=skip).│ -│(TEXT)                                                                         │ -│[default: latest]                                                              │ -│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed      │ -│(default = selected).                                                          │ -│(selected | all | skip | remove)                                               │ -│[default: selected]                                                            │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--verbose-vPrint verbose information about performed steps.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze start-airflow [OPTIONS] [EXTRA_ARGS]... + +Enter breeze environment and starts all Airflow components in the tmux session. Compile assets if contents of www  +directory changed. + +╭─ Basic flags ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.7< | 3.8 | 3.9 | 3.10 | 3.11)                           │ +│[default: 3.7]                                              │ +│--load-example-dags-eEnable configuration to load example DAGs when starting Airflow.│ +│--load-default-connections-cEnable configuration to load default connections when starting Airflow.│ +│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ +│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64)│ +│--postgres-version-PVersion of Postgres used.(>10< | 11 | 12 | 13 | 14)[default: 10]│ +│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ +│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ +│--integrationIntegration(s) to enable when running (can be more than one).                       │ +│(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino│ +│| all)                                                                              │ +│--forward-credentials-fForward local credentials to container when running.│ +│--db-reset-dReset DB when entering the container.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Asset compilation options ──────────────────────────────────────────────────────────────────────────────────────────╮ +│--skip-asset-compilationSkips compilation of assets when starting airflow even if the content of www changed    │ +│(mutually exclusive with --dev-mode).                                                   │ +│--dev-modeStarts webserver in dev mode (assets are always recompiled in this case when starting)  │ +│(mutually exclusive with --skip-asset-compilation).                                     │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced flag for running ──────────────────────────────────────────────────────────────────────────────────────────╮ +│--use-airflow-versionUse (reinstall at entry) Airflow version from PyPI. It can also be `none`,     │ +│`wheel`, or `sdist` if Airflow should be removed, installed from wheel packages│ +│or sdist packages available in dist folder respectively. Implies               │ +│--mount-sources `remove`.                                                      │ +│(none | wheel | sdist | <airflow_version>)                                     │ +│--airflow-constraints-referenceConstraint reference to use. Useful with --use-airflow-version parameter to    │ +│specify constraints for the installed version and to find newer dependencies   │ +│(TEXT)                                                                         │ +│--airflow-extrasAirflow extras to install when --use-airflow-version is used(TEXT)│ +│--use-packages-from-distInstall all found packages (--package-format determines type) from 'dist'      │ +│folder when entering breeze.                                                   │ +│--package-formatFormat of packages that should be installed from dist.(wheel | sdist)│ +│[default: wheel]                                      │ +│--force-buildForce image build no matter if it is determined as needed.│ +│--image-tag-tTag of the image which is used to run the image (implies --mount-sources=skip).│ +│(TEXT)                                                                         │ +│[default: latest]                                                              │ +│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed      │ +│(default = selected).                                                          │ +│(selected | all | skip | remove)                                               │ +│[default: selected]                                                            │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--verbose-vPrint verbose information about performed steps.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_testing.svg b/images/breeze/output_testing.svg index a22633da5fb84..ed5d5d1ccb92f 100644 --- a/images/breeze/output_testing.svg +++ b/images/breeze/output_testing.svg @@ -35,8 +35,8 @@ .terminal-3208273230-r1 { fill: #c5c8c6;font-weight: bold } .terminal-3208273230-r2 { fill: #c5c8c6 } .terminal-3208273230-r3 { fill: #d0b344;font-weight: bold } -.terminal-3208273230-r4 { fill: #868887 } -.terminal-3208273230-r5 { fill: #68a0b3;font-weight: bold } +.terminal-3208273230-r4 { fill: #68a0b3;font-weight: bold } +.terminal-3208273230-r5 { fill: #868887 } .terminal-3208273230-r6 { fill: #98a84b;font-weight: bold } @@ -93,18 +93,18 @@ -Usage: breeze testing [OPTIONS] COMMAND [ARGS]... +Usage: breeze testing [OPTIONS] COMMAND [ARGS]... Tools that developers can use to run tests -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Testing ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│tests                                    Run the specified unit test targets.                                     │ -│helm-tests                               Run Helm chart tests.                                                    │ -│docker-compose-tests                     Run docker-compose tests.                                                │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Testing ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│tests                                    Run the specified unit test targets.                                     │ +│helm-tests                               Run Helm chart tests.                                                    │ +│docker-compose-tests                     Run docker-compose tests.                                                │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_testing_docker-compose-tests.svg b/images/breeze/output_testing_docker-compose-tests.svg index c521157a63cab..93b43a5ed9058 100644 --- a/images/breeze/output_testing_docker-compose-tests.svg +++ b/images/breeze/output_testing_docker-compose-tests.svg @@ -19,113 +19,113 @@ font-weight: 700; } - .terminal-128845089-matrix { + .terminal-950076864-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-128845089-title { + .terminal-950076864-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-128845089-r1 { fill: #c5c8c6;font-weight: bold } -.terminal-128845089-r2 { fill: #c5c8c6 } -.terminal-128845089-r3 { fill: #d0b344;font-weight: bold } -.terminal-128845089-r4 { fill: #868887 } -.terminal-128845089-r5 { fill: #68a0b3;font-weight: bold } -.terminal-128845089-r6 { fill: #98a84b;font-weight: bold } -.terminal-128845089-r7 { fill: #8d7b39 } + .terminal-950076864-r1 { fill: #c5c8c6;font-weight: bold } +.terminal-950076864-r2 { fill: #c5c8c6 } +.terminal-950076864-r3 { fill: #d0b344;font-weight: bold } +.terminal-950076864-r4 { fill: #68a0b3;font-weight: bold } +.terminal-950076864-r5 { fill: #868887 } +.terminal-950076864-r6 { fill: #98a84b;font-weight: bold } +.terminal-950076864-r7 { fill: #8d7b39 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - Command: testing docker-compose-tests + Command: testing docker-compose-tests - + - - -Usage: breeze testing docker-compose-tests [OPTIONS] [EXTRA_PYTEST_ARGS]... - -Run docker-compose tests. - -╭─ Docker-compose tests flag ──────────────────────────────────────────────────────────────────────────────────────────╮ -│--image-name-nName of the image to verify (overrides --python and --image-tag).(TEXT)│ -│--image-tag-tTag of the image which is used to run the image (implies --mount-sources=skip).(TEXT)│ -│[default: latest]                                                              │ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze testing docker-compose-tests [OPTIONS] [EXTRA_PYTEST_ARGS]... + +Run docker-compose tests. + +╭─ Docker-compose tests flag ──────────────────────────────────────────────────────────────────────────────────────────╮ +│--image-name-nName of the image to verify (overrides --python and --image-tag).(TEXT)│ +│--image-tag-tTag of the image which is used to run the image (implies --mount-sources=skip).(TEXT)│ +│[default: latest]                                                              │ +│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10 | 3.11)│ +│[default: 3.7]                                              │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output_testing_tests.svg b/images/breeze/output_testing_tests.svg index 1b25d4efa6221..f7d4506f60dc7 100644 --- a/images/breeze/output_testing_tests.svg +++ b/images/breeze/output_testing_tests.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - Command: testing tests + Command: testing tests - + - - -Usage: breeze testing tests [OPTIONS] [EXTRA_PYTEST_ARGS]... - -Run the specified unit test targets. - -╭─ Basic flag for tests command ───────────────────────────────────────────────────────────────────────────────────────╮ -│--integrationIntegration(s) to enable when running (can be more than one).                              │ -│(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino | all)│ -│--test-typeType of test to run. Note that with Providers, you can also specify which provider tests    │ -│should be run - for example --test-type "Providers[airbyte,http]"                           │ -│(All | API | Always | CLI | Core | Integration | Other | Providers | WWW | Helm | Postgres |│ -│MySQL | Quarantine)                                                                         │ -│--test-timeoutTest timeout. Set the pytest setup, execution and teardown timeouts to this value│ -│(INTEGER RANGE)                                                                  │ -│[default: 60; x>=0]                                                              │ -│--db-reset-dReset DB when entering the container.│ -│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ -│--python-pPython major/minor version used in Airflow image for images.(>3.7< | 3.8 | 3.9 | 3.10)│ -│[default: 3.7]                                              │ -│--postgres-version-PVersion of Postgres used.(>10< | 11 | 12 | 13 | 14)[default: 10]│ -│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ -│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options for parallel test commands ─────────────────────────────────────────────────────────────────────────────────╮ -│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ -│--parallelismMaximum number of processes to use while running the operation in parallel.│ -│(INTEGER RANGE)                                                            │ -│[default: 4; 1<=x<=8]                                                      │ -│--test-typesSpace separated list of test types used for testing in parallel.(TEXT)│ -│[default: API Always CLI Core Integration Other Providers WWW]  │ -│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ -│--debug-resourcesWhether to show resource information while running in parallel.│ -│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced flag for tests command ────────────────────────────────────────────────────────────────────────────────────╮ -│--image-tag-tTag of the image which is used to run the image (implies --mount-sources=skip).(TEXT)│ -│[default: latest]                                                              │ -│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed (default = selected).│ -│(selected | all | skip | remove)                                                               │ -│[default: selected]                                                                            │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--verbose-vPrint verbose information about performed steps.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze testing tests [OPTIONS] [EXTRA_PYTEST_ARGS]... + +Run the specified unit test targets. + +╭─ Basic flag for tests command ───────────────────────────────────────────────────────────────────────────────────────╮ +│--integrationIntegration(s) to enable when running (can be more than one).                              │ +│(cassandra | kerberos | mongo | openldap | pinot | rabbitmq | redis | statsd | trino | all)│ +│--test-typeType of test to run. Note that with Providers, you can also specify which provider tests    │ +│should be run - for example --test-type "Providers[airbyte,http]"                           │ +│(All | API | Always | CLI | Core | Integration | Other | Providers | WWW | Helm | Postgres |│ +│MySQL | Quarantine)                                                                         │ +│--test-timeoutTest timeout. Set the pytest setup, execution and teardown timeouts to this value│ +│(INTEGER RANGE)                                                                  │ +│[default: 60; x>=0]                                                              │ +│--db-reset-dReset DB when entering the container.│ +│--backend-bDatabase backend to use.(>sqlite< | mysql | postgres | mssql)[default: sqlite]│ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.7< | 3.8 | 3.9 | 3.10 | 3.11)                           │ +│[default: 3.7]                                              │ +│--postgres-version-PVersion of Postgres used.(>10< | 11 | 12 | 13 | 14)[default: 10]│ +│--mysql-version-MVersion of MySQL used.(>5.7< | 8)[default: 5.7]│ +│--mssql-version-SVersion of MsSQL used.(>2017-latest< | 2019-latest)[default: 2017-latest]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options for parallel test commands ─────────────────────────────────────────────────────────────────────────────────╮ +│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ +│--parallelismMaximum number of processes to use while running the operation in parallel.│ +│(INTEGER RANGE)                                                            │ +│[default: 4; 1<=x<=8]                                                      │ +│--test-typesSpace separated list of test types used for testing in parallel.(TEXT)│ +│[default: API Always CLI Core Integration Other Providers WWW]  │ +│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ +│--debug-resourcesWhether to show resource information while running in parallel.│ +│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced flag for tests command ────────────────────────────────────────────────────────────────────────────────────╮ +│--image-tag-tTag of the image which is used to run the image (implies --mount-sources=skip).(TEXT)│ +│[default: latest]                                                              │ +│--mount-sourcesChoose scope of local sources that should be mounted, skipped, or removed (default = selected).│ +│(selected | all | skip | remove)                                                               │ +│[default: selected]                                                                            │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--verbose-vPrint verbose information about performed steps.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/scripts/ci/pre_commit/pre_commit_build_providers_dependencies.py b/scripts/ci/pre_commit/pre_commit_build_providers_dependencies.py index b3782955a64d5..8ecdb513906b1 100755 --- a/scripts/ci/pre_commit/pre_commit_build_providers_dependencies.py +++ b/scripts/ci/pre_commit/pre_commit_build_providers_dependencies.py @@ -45,9 +45,6 @@ warnings: list[str] = [] errors: list[str] = [] -CROSS_PROVIDERS_DEPS = "cross-providers-deps" -DEPS = "deps" - ALL_DEPENDENCIES: dict[str, dict[str, list[str]]] = defaultdict(lambda: defaultdict(list)) ALL_PROVIDERS: dict[str, dict[str, Any]] = defaultdict(lambda: defaultdict()) @@ -160,7 +157,7 @@ def check_if_different_provider_used(file_path: Path) -> None: if imported_provider is not None and imported_provider not in ALL_PROVIDERS: warnings.append(f"The provider {imported_provider} from {file_path} cannot be found.") elif imported_provider and file_provider != imported_provider: - ALL_DEPENDENCIES[file_provider][CROSS_PROVIDERS_DEPS].append(imported_provider) + ALL_DEPENDENCIES[file_provider]["cross-providers-deps"].append(imported_provider) if __name__ == "__main__": @@ -173,7 +170,7 @@ def check_if_different_provider_used(file_path: Path) -> None: check_if_different_provider_used(file) for provider, provider_yaml_content in ALL_PROVIDERS.items(): - ALL_DEPENDENCIES[provider][DEPS].extend(provider_yaml_content["dependencies"]) + ALL_DEPENDENCIES[provider]["deps"].extend(provider_yaml_content["dependencies"]) if warnings: console.print("[yellow]Warnings!\n") @@ -187,9 +184,13 @@ def check_if_different_provider_used(file_path: Path) -> None: console.print(f"[bright_blue]Total: {len(errors)} errors.") unique_sorted_dependencies: dict[str, dict[str, list[str]]] = defaultdict(dict) for key in sorted(ALL_DEPENDENCIES.keys()): - unique_sorted_dependencies[key][DEPS] = sorted(ALL_DEPENDENCIES[key][DEPS]) - unique_sorted_dependencies[key][CROSS_PROVIDERS_DEPS] = sorted( - set(ALL_DEPENDENCIES[key][CROSS_PROVIDERS_DEPS]) + unique_sorted_dependencies[key]["deps"] = sorted(ALL_DEPENDENCIES[key]["deps"]) + unique_sorted_dependencies[key]["cross-providers-deps"] = sorted( + set(ALL_DEPENDENCIES[key]["cross-providers-deps"]) + ) + excluded_versions = ALL_PROVIDERS[key].get("excluded-python-versions") + unique_sorted_dependencies[key]["excluded-python-versions"] = ( + excluded_versions if excluded_versions else [] ) if errors: console.print() diff --git a/setup.cfg b/setup.cfg index f6ba7e946e6a9..1dd0f3b88d3e5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -39,6 +39,7 @@ classifiers = Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 + Programming Language :: Python :: 3.11 Topic :: System :: Monitoring Framework :: Apache Airflow project_urls = diff --git a/setup.py b/setup.py index c340cc294e3c0..3a8ea4557e98a 100644 --- a/setup.py +++ b/setup.py @@ -54,8 +54,7 @@ AIRFLOW_SOURCES_ROOT = Path(__file__).parent.resolve() PROVIDERS_ROOT = AIRFLOW_SOURCES_ROOT / "airflow" / "providers" -CROSS_PROVIDERS_DEPS = "cross-providers-deps" -DEPS = "deps" +CURRENT_PYTHON_VERSION = f"{sys.version_info.major}.{sys.version_info.minor}" # @@ -65,7 +64,17 @@ # def fill_provider_dependencies() -> dict[str, dict[str, list[str]]]: try: - return json.loads((AIRFLOW_SOURCES_ROOT / "generated" / "provider_dependencies.json").read_text()) + dependencies = json.loads( + (AIRFLOW_SOURCES_ROOT / "generated" / "provider_dependencies.json").read_text() + ) + for key in list(dependencies.keys()): + if CURRENT_PYTHON_VERSION in dependencies[key]["excluded-python-versions"]: + print( + f"Excluding provider {key} because it is not " + f"compatible with Python {CURRENT_PYTHON_VERSION}" + ) + del dependencies[key] + return dependencies except Exception as e: print(f"Exception while loading provider dependencies {e}") # we can ignore loading dependencies when they are missing - they are only used to generate @@ -411,7 +420,10 @@ def write_version(filename: str = str(AIRFLOW_SOURCES_ROOT / "airflow" / "git_ve def get_provider_dependencies(provider_name: str) -> list[str]: - return PROVIDER_DEPENDENCIES[provider_name][DEPS] + provider_dict = PROVIDER_DEPENDENCIES.get(provider_name) + if provider_dict: + return PROVIDER_DEPENDENCIES[provider_name]["deps"] + return [] def get_unique_dependency_list(req_list_iterable: Iterable[list[str]]): @@ -447,7 +459,7 @@ def get_unique_dependency_list(req_list_iterable: Iterable[list[str]]): ) # Those are all additional extras which do not have their own 'providers' -# The 'apache.atlas' and 'apache.webhdfs' are extras that provide additional libraries +# The 'apache.atlas' and 'apache.webhdfs' are extras that provide additional libraries, # but they do not have separate providers (yet?), they are merely there to add extra libraries # That can be used in custom python/bash operators. ADDITIONAL_EXTRAS_DEPENDENCIES: dict[str, list[str]] = { @@ -477,12 +489,17 @@ def get_unique_dependency_list(req_list_iterable: Iterable[list[str]]): "virtualenv": virtualenv, } +for key in list(CORE_EXTRAS_DEPENDENCIES.keys()): + if not CORE_EXTRAS_DEPENDENCIES[key]: + print(f"Removing extra {key} as it has been excluded") + del CORE_EXTRAS_DEPENDENCIES[key] + EXTRAS_DEPENDENCIES: dict[str, list[str]] = deepcopy(CORE_EXTRAS_DEPENDENCIES) def add_extras_for_all_providers() -> None: for (provider_name, provider_dict) in PROVIDER_DEPENDENCIES.items(): - EXTRAS_DEPENDENCIES[provider_name] = provider_dict[DEPS] + EXTRAS_DEPENDENCIES[provider_name] = provider_dict["deps"] def add_additional_extras() -> None: @@ -539,8 +556,9 @@ def add_extras_for_all_deprecated_aliases() -> None: for alias, extra in EXTRAS_DEPRECATED_ALIASES.items(): dependencies = EXTRAS_DEPENDENCIES.get(extra) if extra != "" else [] if dependencies is None: - raise Exception(f"The extra {extra} is missing for deprecated alias {alias}") - EXTRAS_DEPENDENCIES[alias] = dependencies + print(f"Skipping adding {alias} alias because {extra} has been excluded") + else: + EXTRAS_DEPENDENCIES[alias] = dependencies def add_all_deprecated_provider_packages() -> None: @@ -593,8 +611,12 @@ def add_all_deprecated_provider_packages() -> None: def get_all_db_dependencies() -> list[str]: _all_db_reqs: set[str] = set() for provider in ALL_DB_PROVIDERS: - for req in PROVIDER_DEPENDENCIES[provider][DEPS]: - _all_db_reqs.add(req) + dependencies = PROVIDER_DEPENDENCIES.get(provider) + if dependencies is None: + print(f"Not adding {provider} dependencies to all_db as it has been excluded") + else: + for req in PROVIDER_DEPENDENCIES[provider]["deps"]: + _all_db_reqs.add(req) return list(_all_db_reqs)