From b96e28997556465cb6311c47a0dbed97848c405b Mon Sep 17 00:00:00 2001 From: Andrew McKenzie <70337292+url54@users.noreply.github.com> Date: Thu, 11 Jan 2024 16:39:57 -0500 Subject: [PATCH 1/6] Update entrypoint.sh Added functionality to ensure "package_requirements.txt" doesn't reference a public URL cause packages to not get installed. --- docker/script/entrypoint.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/script/entrypoint.sh b/docker/script/entrypoint.sh index 24c40870b..220721cca 100644 --- a/docker/script/entrypoint.sh +++ b/docker/script/entrypoint.sh @@ -41,10 +41,11 @@ package_requirements() { if [[ -e "$AIRFLOW_HOME/$REQUIREMENTS_FILE" ]]; then echo "Packaging requirements.txt into plugins" pip3 download -r "$AIRFLOW_HOME/$REQUIREMENTS_FILE" -d "$AIRFLOW_HOME/plugins" - cd "$AIRFLOW_HOME/plugins" + cd "$AIRFLOW_HOME/plugins" zip "$AIRFLOW_HOME/requirements/plugins.zip" * printf '%s\n%s\n' "--no-index" "$(cat $AIRFLOW_HOME/$REQUIREMENTS_FILE)" > "$AIRFLOW_HOME/requirements/packaged_requirements.txt" - printf '%s\n%s\n' "--find-links /usr/local/airflow/plugins" "$(cat $AIRFLOW_HOME/requirements/packaged_requirements.txt)" > "$AIRFLOW_HOME/requirements/packaged_requirements.txt" + printf '%s\n%s\n%s\n%s\n' "--find-links /usr/local/airflow/plugins" "--constraint \"$AIRFLOW_HOME/dags/constraints.txt\"" "$(cat $AIRFLOW_HOME/requirements/packaged_requirements.txt | grep -v '^--constraint .https://*')" > "$AIRFLOW_HOME/requirements/packaged_requirements.txt" + wget "https://raw.githubusercontent.com/apache/airflow/constraints-${AIRFLOW_VERSION}/constraints-${PYTHON_VERSION%.*}.txt" -O $AIRFLOW_HOME/requirements/constraints.txt fi } From 517643a2953e023094aadf3be0ce2643b768f332 Mon Sep 17 00:00:00 2001 From: Andrew McKenzie <70337292+url54@users.noreply.github.com> Date: Thu, 11 Jan 2024 16:40:41 -0500 Subject: [PATCH 2/6] Update Dockerfile Add AIRFLOW_VERSION environment variable to ensure it is usable in entrypoint.sh. --- docker/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index a0d91798a..782f399e3 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -21,6 +21,7 @@ ARG INDEX_URL="" ENV AIRFLOW_HOME=${AIRFLOW_USER_HOME} ENV PATH="$PATH:/usr/local/airflow/.local/bin:/root/.local/bin:/usr/local/airflow/.local/lib/python3.10/site-packages" ENV PYTHON_VERSION=3.11.6 +ENV AIRFLOW_VERSION=2.7.2 COPY script/bootstrap.sh /bootstrap.sh COPY script/systemlibs.sh /systemlibs.sh From 6eac8a5d2bb672e7675cf3bb0ce908b50fbd2a1d Mon Sep 17 00:00:00 2001 From: Andrew McKenzie <70337292+url54@users.noreply.github.com> Date: Thu, 11 Jan 2024 16:43:36 -0500 Subject: [PATCH 3/6] Update entrypoint.sh Fixed spacing. --- docker/script/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/script/entrypoint.sh b/docker/script/entrypoint.sh index 220721cca..ea6faf0e8 100644 --- a/docker/script/entrypoint.sh +++ b/docker/script/entrypoint.sh @@ -41,7 +41,7 @@ package_requirements() { if [[ -e "$AIRFLOW_HOME/$REQUIREMENTS_FILE" ]]; then echo "Packaging requirements.txt into plugins" pip3 download -r "$AIRFLOW_HOME/$REQUIREMENTS_FILE" -d "$AIRFLOW_HOME/plugins" - cd "$AIRFLOW_HOME/plugins" + cd "$AIRFLOW_HOME/plugins" zip "$AIRFLOW_HOME/requirements/plugins.zip" * printf '%s\n%s\n' "--no-index" "$(cat $AIRFLOW_HOME/$REQUIREMENTS_FILE)" > "$AIRFLOW_HOME/requirements/packaged_requirements.txt" printf '%s\n%s\n%s\n%s\n' "--find-links /usr/local/airflow/plugins" "--constraint \"$AIRFLOW_HOME/dags/constraints.txt\"" "$(cat $AIRFLOW_HOME/requirements/packaged_requirements.txt | grep -v '^--constraint .https://*')" > "$AIRFLOW_HOME/requirements/packaged_requirements.txt" From 5f41feac041e8ad9d22f31f879c74ce7110aa0b4 Mon Sep 17 00:00:00 2001 From: Andrew McKenzie <70337292+url54@users.noreply.github.com> Date: Thu, 11 Jan 2024 16:50:27 -0500 Subject: [PATCH 4/6] Update README.md Added contextual information to the readme, about this update and the constraints.txt file. --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 49b402bd1..78cb9540d 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,7 @@ For example usage see [Installing Python dependencies using PyPi.org Requirement - There is a directory at the root of this repository called plugins. - In this directory, create a file for your new custom plugin. - Add any Python dependencies to `requirements/requirements.txt`. +- Adds a local `constraints.txt` file to the `requirements/` directory, that you can add to your S3 bucket's dag folder. Ensuring you have the complete package: packaged_requirements.txt, plugins.zip, and constraints.txt, to build your private Environment **Note**: this step assumes you have a DAG that corresponds to the custom plugin. For example usage [MWAA Code Examples](https://docs.aws.amazon.com/mwaa/latest/userguide/sample-code.html). @@ -157,6 +158,7 @@ For example usage see [Installing Python dependencies using PyPi.org Requirement - Learn how to upload the requirements.txt file to your Amazon S3 bucket in [Installing Python dependencies](https://docs.aws.amazon.com/mwaa/latest/userguide/working-dags-dependencies.html). - Learn how to upload the DAG code to the dags folder in your Amazon S3 bucket in [Adding or updating DAGs](https://docs.aws.amazon.com/mwaa/latest/userguide/configuring-dag-folder.html). - Learn more about how to upload the plugins.zip file to your Amazon S3 bucket in [Installing custom plugins](https://docs.aws.amazon.com/mwaa/latest/userguide/configuring-dag-import-plugins.html). +- If using `package-requirements`, make sure you upload the `constraints.txt` file that is auto-generated for you in the `requirements/` directory to your S3 bucket's dag folder. ## FAQs From 289efd83bb7fd4a5b946dc39d02c89931a5b1290 Mon Sep 17 00:00:00 2001 From: charlielu05 Date: Wed, 24 Jan 2024 04:14:36 +0000 Subject: [PATCH 5/6] Add constraints.txt to plugins.zip, updated readme --- README.md | 4 ++-- docker/script/entrypoint.sh | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 78cb9540d..0655fe999 100644 --- a/README.md +++ b/README.md @@ -139,7 +139,8 @@ For example usage see [Installing Python dependencies using PyPi.org Requirement - There is a directory at the root of this repository called plugins. - In this directory, create a file for your new custom plugin. - Add any Python dependencies to `requirements/requirements.txt`. -- Adds a local `constraints.txt` file to the `requirements/` directory, that you can add to your S3 bucket's dag folder. Ensuring you have the complete package: packaged_requirements.txt, plugins.zip, and constraints.txt, to build your private Environment +- Adds a local `constraints.txt` file to the `plugins/` directory, this is zipped together into the `plugins.zip` artefact. +- Creates a new `packaged_requirements.txt` file with the correct configuration for `--find-links` and `--constraint`. This file should be the one you rename to `requirements.txt` and upload to S3 to be used by MWAA. **Note**: this step assumes you have a DAG that corresponds to the custom plugin. For example usage [MWAA Code Examples](https://docs.aws.amazon.com/mwaa/latest/userguide/sample-code.html). @@ -158,7 +159,6 @@ For example usage see [Installing Python dependencies using PyPi.org Requirement - Learn how to upload the requirements.txt file to your Amazon S3 bucket in [Installing Python dependencies](https://docs.aws.amazon.com/mwaa/latest/userguide/working-dags-dependencies.html). - Learn how to upload the DAG code to the dags folder in your Amazon S3 bucket in [Adding or updating DAGs](https://docs.aws.amazon.com/mwaa/latest/userguide/configuring-dag-folder.html). - Learn more about how to upload the plugins.zip file to your Amazon S3 bucket in [Installing custom plugins](https://docs.aws.amazon.com/mwaa/latest/userguide/configuring-dag-import-plugins.html). -- If using `package-requirements`, make sure you upload the `constraints.txt` file that is auto-generated for you in the `requirements/` directory to your S3 bucket's dag folder. ## FAQs diff --git a/docker/script/entrypoint.sh b/docker/script/entrypoint.sh index ea6faf0e8..b06eb26b6 100644 --- a/docker/script/entrypoint.sh +++ b/docker/script/entrypoint.sh @@ -41,11 +41,11 @@ package_requirements() { if [[ -e "$AIRFLOW_HOME/$REQUIREMENTS_FILE" ]]; then echo "Packaging requirements.txt into plugins" pip3 download -r "$AIRFLOW_HOME/$REQUIREMENTS_FILE" -d "$AIRFLOW_HOME/plugins" - cd "$AIRFLOW_HOME/plugins" + wget "https://raw.githubusercontent.com/apache/airflow/constraints-${AIRFLOW_VERSION}/constraints-${PYTHON_VERSION%.*}.txt" -O $AIRFLOW_HOME/plugins/constraints.txt + cd "$AIRFLOW_HOME/plugins" zip "$AIRFLOW_HOME/requirements/plugins.zip" * printf '%s\n%s\n' "--no-index" "$(cat $AIRFLOW_HOME/$REQUIREMENTS_FILE)" > "$AIRFLOW_HOME/requirements/packaged_requirements.txt" - printf '%s\n%s\n%s\n%s\n' "--find-links /usr/local/airflow/plugins" "--constraint \"$AIRFLOW_HOME/dags/constraints.txt\"" "$(cat $AIRFLOW_HOME/requirements/packaged_requirements.txt | grep -v '^--constraint .https://*')" > "$AIRFLOW_HOME/requirements/packaged_requirements.txt" - wget "https://raw.githubusercontent.com/apache/airflow/constraints-${AIRFLOW_VERSION}/constraints-${PYTHON_VERSION%.*}.txt" -O $AIRFLOW_HOME/requirements/constraints.txt + printf '%s\n%s\n%s\n%s\n' "--find-links /usr/local/airflow/plugins" "--constraint /usr/local/airflow/plugins/constraints.txt" "$(cat $AIRFLOW_HOME/requirements/packaged_requirements.txt | grep -v '^--constraint .https://*')" > "$AIRFLOW_HOME/requirements/packaged_requirements.txt" fi } From c019a412ac94538fb96bfd6c6664a2d1fafe8fa6 Mon Sep 17 00:00:00 2001 From: Charlie Lu Date: Tue, 23 Apr 2024 11:53:24 +1000 Subject: [PATCH 6/6] changes to reflect PR comments --- docker/Dockerfile | 2 +- docker/script/entrypoint.sh | 4 ++-- requirements/requirements.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 782f399e3..7c03d53f0 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -21,7 +21,7 @@ ARG INDEX_URL="" ENV AIRFLOW_HOME=${AIRFLOW_USER_HOME} ENV PATH="$PATH:/usr/local/airflow/.local/bin:/root/.local/bin:/usr/local/airflow/.local/lib/python3.10/site-packages" ENV PYTHON_VERSION=3.11.6 -ENV AIRFLOW_VERSION=2.7.2 +ENV AIRFLOW_VERSION=${AIRFLOW_VERSION} COPY script/bootstrap.sh /bootstrap.sh COPY script/systemlibs.sh /systemlibs.sh diff --git a/docker/script/entrypoint.sh b/docker/script/entrypoint.sh index b06eb26b6..a7dc27ff3 100644 --- a/docker/script/entrypoint.sh +++ b/docker/script/entrypoint.sh @@ -41,11 +41,11 @@ package_requirements() { if [[ -e "$AIRFLOW_HOME/$REQUIREMENTS_FILE" ]]; then echo "Packaging requirements.txt into plugins" pip3 download -r "$AIRFLOW_HOME/$REQUIREMENTS_FILE" -d "$AIRFLOW_HOME/plugins" - wget "https://raw.githubusercontent.com/apache/airflow/constraints-${AIRFLOW_VERSION}/constraints-${PYTHON_VERSION%.*}.txt" -O $AIRFLOW_HOME/plugins/constraints.txt + wget "https://raw.githubusercontent.com/aws/aws-mwaa-local-runner/v${AIRFLOW_VERSION}/docker/config/constraints.txt" -O $AIRFLOW_HOME/plugins/constraints.txt cd "$AIRFLOW_HOME/plugins" zip "$AIRFLOW_HOME/requirements/plugins.zip" * printf '%s\n%s\n' "--no-index" "$(cat $AIRFLOW_HOME/$REQUIREMENTS_FILE)" > "$AIRFLOW_HOME/requirements/packaged_requirements.txt" - printf '%s\n%s\n%s\n%s\n' "--find-links /usr/local/airflow/plugins" "--constraint /usr/local/airflow/plugins/constraints.txt" "$(cat $AIRFLOW_HOME/requirements/packaged_requirements.txt | grep -v '^--constraint .https://*')" > "$AIRFLOW_HOME/requirements/packaged_requirements.txt" + printf '%s\n%s\n%s\n%s\n' "--find-links /usr/local/airflow/plugins" "--constraint /usr/local/airflow/plugins/constraints.txt" "$(cat $AIRFLOW_HOME/requirements/packaged_requirements.txt | grep -v '^--constraint')" > "$AIRFLOW_HOME/requirements/packaged_requirements.txt" fi } diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 185cda4e4..f94740373 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,4 +1,4 @@ ---constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.7.2/constraints-3.11.txt" +--constraint "https://raw.githubusercontent.com/aws/aws-mwaa-local-runner/v2.7.2/docker/config/constraints.txt" apache-airflow-providers-snowflake==5.0.1 apache-airflow-providers-mysql==5.3.1