diff --git a/README.md b/README.md index 49b402bd1..0655fe999 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,8 @@ For example usage see [Installing Python dependencies using PyPi.org Requirement - There is a directory at the root of this repository called plugins. - In this directory, create a file for your new custom plugin. - Add any Python dependencies to `requirements/requirements.txt`. +- Adds a local `constraints.txt` file to the `plugins/` directory, this is zipped together into the `plugins.zip` artefact. +- Creates a new `packaged_requirements.txt` file with the correct configuration for `--find-links` and `--constraint`. This file should be the one you rename to `requirements.txt` and upload to S3 to be used by MWAA. **Note**: this step assumes you have a DAG that corresponds to the custom plugin. For example usage [MWAA Code Examples](https://docs.aws.amazon.com/mwaa/latest/userguide/sample-code.html). diff --git a/docker/Dockerfile b/docker/Dockerfile index a0d91798a..7c03d53f0 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -21,6 +21,7 @@ ARG INDEX_URL="" ENV AIRFLOW_HOME=${AIRFLOW_USER_HOME} ENV PATH="$PATH:/usr/local/airflow/.local/bin:/root/.local/bin:/usr/local/airflow/.local/lib/python3.10/site-packages" ENV PYTHON_VERSION=3.11.6 +ENV AIRFLOW_VERSION=${AIRFLOW_VERSION} COPY script/bootstrap.sh /bootstrap.sh COPY script/systemlibs.sh /systemlibs.sh diff --git a/docker/script/entrypoint.sh b/docker/script/entrypoint.sh index 24c40870b..a7dc27ff3 100644 --- a/docker/script/entrypoint.sh +++ b/docker/script/entrypoint.sh @@ -41,10 +41,11 @@ package_requirements() { if [[ -e "$AIRFLOW_HOME/$REQUIREMENTS_FILE" ]]; then echo "Packaging requirements.txt into plugins" pip3 download -r "$AIRFLOW_HOME/$REQUIREMENTS_FILE" -d "$AIRFLOW_HOME/plugins" + wget "https://raw.githubusercontent.com/aws/aws-mwaa-local-runner/v${AIRFLOW_VERSION}/docker/config/constraints.txt" -O $AIRFLOW_HOME/plugins/constraints.txt cd "$AIRFLOW_HOME/plugins" zip "$AIRFLOW_HOME/requirements/plugins.zip" * printf '%s\n%s\n' "--no-index" "$(cat $AIRFLOW_HOME/$REQUIREMENTS_FILE)" > "$AIRFLOW_HOME/requirements/packaged_requirements.txt" - printf '%s\n%s\n' "--find-links /usr/local/airflow/plugins" "$(cat $AIRFLOW_HOME/requirements/packaged_requirements.txt)" > "$AIRFLOW_HOME/requirements/packaged_requirements.txt" + printf '%s\n%s\n%s\n%s\n' "--find-links /usr/local/airflow/plugins" "--constraint /usr/local/airflow/plugins/constraints.txt" "$(cat $AIRFLOW_HOME/requirements/packaged_requirements.txt | grep -v '^--constraint')" > "$AIRFLOW_HOME/requirements/packaged_requirements.txt" fi } diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 185cda4e4..f94740373 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,4 +1,4 @@ ---constraint "https://raw.githubusercontent.com/apache/airflow/constraints-2.7.2/constraints-3.11.txt" +--constraint "https://raw.githubusercontent.com/aws/aws-mwaa-local-runner/v2.7.2/docker/config/constraints.txt" apache-airflow-providers-snowflake==5.0.1 apache-airflow-providers-mysql==5.3.1