From b665f04e8759c3585f25b69a8fc3b1b7f6279f1c Mon Sep 17 00:00:00 2001 From: Alex Morega Date: Mon, 27 Nov 2023 18:57:39 +0200 Subject: [PATCH] Switch to pip-tools version pinning --- README.md | 8 ++ constraints.txt | 53 --------- docker/Dockerfile | 12 +- docker/constraints.txt | 1 - docker/requirements.txt | 3 - requirements.txt => requirements/base.in | 21 ++-- requirements/base.txt | 135 +++++++++++++++++++++++ requirements/compile | 9 ++ requirements/dev.in | 5 + requirements/dev.txt | 50 +++++++++ requirements/docker.in | 3 + requirements/docker.txt | 12 ++ 12 files changed, 235 insertions(+), 77 deletions(-) delete mode 100644 constraints.txt delete mode 100644 docker/constraints.txt delete mode 100644 docker/requirements.txt rename requirements.txt => requirements/base.in (71%) create mode 100644 requirements/base.txt create mode 100755 requirements/compile create mode 100644 requirements/dev.in create mode 100644 requirements/dev.txt create mode 100644 requirements/docker.in create mode 100644 requirements/docker.txt diff --git a/README.md b/README.md index 3261d7b3..c4878607 100644 --- a/README.md +++ b/README.md @@ -121,6 +121,14 @@ This repo contains code for both the IETF and IAB websites, which are intended t - In settings -> layout settings (http://localhost:8001/admin/settings/utils/layoutsettings/2/), select your new site and make sure that the base template is set to IAB - Header and footer links are populated in the same way as the IETF website. The header contains pages that have the 'show in menu' checkbox ticked in the 'promote' tab. Footer links are set in settings -> footer links. +## Upgrading dependencies + +Dependencies are managed using [pip-tools](https://pip-tools.readthedocs.io/en/stable/). They are specified in `requirements/*.in` and version-pinned in `requirements/*.txt`. To update the pins, run: + +```sh +docker compose run --rm application requirements/compile -U +``` + ## Deployment Production: [IETF](https://www.ietf.org/), [IAB](https://temporary.iab.org/) diff --git a/constraints.txt b/constraints.txt deleted file mode 100644 index 06ce0209..00000000 --- a/constraints.txt +++ /dev/null @@ -1,53 +0,0 @@ -anyascii==0.3.2 -asgiref==3.7.2 -beautifulsoup4==4.11.2 -bleach==4.1.0 -certifi==2023.5.7 -chardet==5.1.0 -charset-normalizer==3.2.0 -dj-database-url==0.5.0 -Django==3.2.20 -django-analytical==3.1.0 -django-appconf==1.0.5 -django-compressor==4.4 -django-filter==22.1 -django-libsass==0.9 -django-modelcluster==6.0 -django-permissionedforms==0.1 -django-taggit==3.1.0 -django-treebeard==4.7 -djangorestframework==3.14.0 -draftjs-exporter==2.1.7 -et-xmlfile==1.1.0 -future==0.18.3 -gunicorn==20.1.0 -html5lib==1.1 -idna==3.4 -l18n==2021.3 -libsass==0.22.0 -Markdown==3.4.3 -mbstrdecoder==1.1.3 -mod-wsgi==4.9.4 -openpyxl==3.1.2 -packaging==23.1 -Pillow==9.5.0 -psycopg2-binary==2.9.6 -Pygments==2.15.1 -pytz==2023.3 -rcssmin==1.1.1 -requests==2.31.0 -rjsmin==1.2.1 -six==1.16.0 -soupsieve==2.4.1 -sqlparse==0.4.4 -telepath==0.3.1 -tqdm==4.65.0 -typed-environment-configuration==0.1.5 -typepy==1.3.0 -typing_extensions==4.7.1 -urllib3==2.0.3 -wagtail==4.1.6 -wagtail-markdown==0.11.0 -wagtail-orderable==1.2.0 -webencodings==0.5.1 -Willow==1.4.1 diff --git a/docker/Dockerfile b/docker/Dockerfile index 2eb2a3be..b35d82f7 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -39,9 +39,7 @@ RUN apk update && apk add postgresql-libs postgresql-client libevent libjpeg ope WORKDIR /app # Copy Python reqs to benefit from Docker caching -COPY docker/requirements.txt docker/constraints.txt /app/docker/ -COPY requirements.txt /app/requirements/main.txt -COPY constraints.txt /app/requirements/constraints.txt +COPY requirements ./requirements # Install build dependencies, then Python requirements, then remove build dependencies # to reduce the resulting image size @@ -52,8 +50,7 @@ RUN apk update && apk add --virtual build-deps make gcc g++ musl-dev apache2-dev openjpeg-dev \ libevent-dev && \ cd /app && \ - pip3 install -r docker/requirements.txt && \ - pip3 install -r requirements/main.txt && \ + pip3 install -r requirements/base.txt -r requirements/docker.txt && \ apk del build-deps build-headers RUN adduser -S www -u 1000 && chown -R www /app @@ -88,7 +85,8 @@ COPY docker/init-dev.sh /app/docker/ ADD https://raw.githubusercontent.com/mrako/wait-for/d9699cb9fe8a4622f05c4ee32adf2fd93239d005/wait-for /usr/local/bin/ USER root -RUN pip3 install pytest-django pytest-cov +RUN apk add --no-cache bash +RUN pip3 install -r requirements/dev.txt RUN chmod +rx /usr/local/bin/wait-for USER www @@ -107,7 +105,7 @@ COPY docker/init-test.sh /app/docker/ ADD https://raw.githubusercontent.com/mrako/wait-for/d9699cb9fe8a4622f05c4ee32adf2fd93239d005/wait-for /usr/local/bin/ USER root -RUN pip3 install pytest-django pytest-cov +RUN pip3 install -r requirements/dev.txt RUN chmod +rx /usr/local/bin/wait-for USER www diff --git a/docker/constraints.txt b/docker/constraints.txt deleted file mode 100644 index 9d41f264..00000000 --- a/docker/constraints.txt +++ /dev/null @@ -1 +0,0 @@ -gunicorn==20.1.0 diff --git a/docker/requirements.txt b/docker/requirements.txt deleted file mode 100644 index 8e9871db..00000000 --- a/docker/requirements.txt +++ /dev/null @@ -1,3 +0,0 @@ -# WebServer --c constraints.txt -gunicorn>=20.1.0 diff --git a/requirements.txt b/requirements/base.in similarity index 71% rename from requirements.txt rename to requirements/base.in index 00713ef5..bccfcf80 100644 --- a/requirements.txt +++ b/requirements/base.in @@ -1,18 +1,13 @@ -# link packages version lock down file --c constraints.txt - -Django>=3.0,<3.3 -psycopg2-binary>=2.7.5 -wagtail>=4.1,<4.2 +dj-database-url>=0.5.0,<0.6 django-libsass>=0.8 -libsass>=0.8.3 -django_compressor>=2.2 +django>=3.0,<3.3 django_analytical>=2.5 +django_compressor>=2.2 +libsass>=0.8.3 +psycopg2-binary>=2.7.5 +pymemcache tqdm>=3.7.0 +typed-environment-configuration>=0.1.3,<0.2 wagtail-markdown wagtail-orderable -pymemcache - -# env var configuration -typed-environment-configuration>=0.1.3,<0.2 -dj-database-url>=0.5.0,<0.6 +wagtail>=4.1,<4.2 diff --git a/requirements/base.txt b/requirements/base.txt new file mode 100644 index 00000000..dc3b57af --- /dev/null +++ b/requirements/base.txt @@ -0,0 +1,135 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile base.in +# +anyascii==0.3.2 + # via wagtail +asgiref==3.7.2 + # via django +beautifulsoup4==4.11.2 + # via wagtail +bleach==4.1.0 + # via wagtail-markdown +certifi==2023.11.17 + # via requests +chardet==5.2.0 + # via mbstrdecoder +charset-normalizer==3.3.2 + # via requests +defusedxml==0.7.1 + # via willow +dj-database-url==0.5.0 + # via -r base.in +django==3.2.23 + # via + # -r base.in + # django-appconf + # django-filter + # django-modelcluster + # django-permissionedforms + # django-taggit + # django-treebeard + # djangorestframework + # wagtail +django-analytical==3.1.0 + # via -r base.in +django-appconf==1.0.6 + # via django-compressor +django-compressor==4.4 + # via + # -r base.in + # django-libsass +django-filter==22.1 + # via wagtail +django-libsass==0.9 + # via -r base.in +django-modelcluster==6.1 + # via wagtail +django-permissionedforms==0.1 + # via wagtail +django-taggit==3.1.0 + # via wagtail +django-treebeard==4.7 + # via wagtail +djangorestframework==3.14.0 + # via wagtail +draftjs-exporter==2.1.7 + # via wagtail +et-xmlfile==1.1.0 + # via openpyxl +filetype==1.2.0 + # via willow +html5lib==1.1 + # via wagtail +idna==3.6 + # via requests +l18n==2021.3 + # via wagtail +libsass==0.22.0 + # via + # -r base.in + # django-libsass +markdown==3.5.1 + # via wagtail-markdown +mbstrdecoder==1.1.3 + # via typepy +openpyxl==3.1.2 + # via wagtail +packaging==23.2 + # via bleach +pillow==10.1.0 + # via wagtail +psycopg2-binary==2.9.9 + # via -r base.in +pymemcache==4.0.0 + # via -r base.in +pytz==2023.3.post1 + # via + # django + # django-modelcluster + # djangorestframework + # l18n +rcssmin==1.1.1 + # via django-compressor +requests==2.31.0 + # via wagtail +rjsmin==1.2.1 + # via django-compressor +six==1.16.0 + # via + # bleach + # html5lib + # l18n +soupsieve==2.5 + # via beautifulsoup4 +sqlparse==0.4.4 + # via django +telepath==0.3.1 + # via wagtail +tqdm==4.66.1 + # via -r base.in +typed-environment-configuration==0.1.5 + # via -r base.in +typepy==1.3.0 + # via typed-environment-configuration +typing-extensions==4.8.0 + # via asgiref +urllib3==2.1.0 + # via requests +wagtail==4.1.9 + # via + # -r base.in + # wagtail-markdown + # wagtail-orderable +wagtail-markdown==0.11.1 + # via -r base.in +wagtail-orderable==1.2.0 + # via -r base.in +webencodings==0.5.1 + # via + # bleach + # html5lib +willow==1.6.3 + # via wagtail diff --git a/requirements/compile b/requirements/compile new file mode 100755 index 00000000..a737920f --- /dev/null +++ b/requirements/compile @@ -0,0 +1,9 @@ +#!/bin/bash + +set -euo pipefail +cd "$( dirname "${BASH_SOURCE[0]}" )" + +set -x +pip-compile base.in "$@" +pip-compile docker.in "$@" +pip-compile dev.in "$@" diff --git a/requirements/dev.in b/requirements/dev.in new file mode 100644 index 00000000..03bac4d6 --- /dev/null +++ b/requirements/dev.in @@ -0,0 +1,5 @@ +-c base.txt + +pip-tools +pytest-cov +pytest-django diff --git a/requirements/dev.txt b/requirements/dev.txt new file mode 100644 index 00000000..5366e4de --- /dev/null +++ b/requirements/dev.txt @@ -0,0 +1,50 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile dev.in +# +build==1.0.3 + # via pip-tools +click==8.1.7 + # via pip-tools +coverage[toml]==7.3.2 + # via + # coverage + # pytest-cov +exceptiongroup==1.2.0 + # via pytest +iniconfig==2.0.0 + # via pytest +packaging==23.2 + # via + # -c base.txt + # build + # pytest +pip-tools==7.3.0 + # via -r dev.in +pluggy==1.3.0 + # via pytest +pyproject-hooks==1.0.0 + # via build +pytest==7.4.3 + # via + # pytest-cov + # pytest-django +pytest-cov==4.1.0 + # via -r dev.in +pytest-django==4.7.0 + # via -r dev.in +tomli==2.0.1 + # via + # build + # coverage + # pip-tools + # pyproject-hooks + # pytest +wheel==0.42.0 + # via pip-tools + +# The following packages are considered to be unsafe in a requirements file: +# pip +# setuptools diff --git a/requirements/docker.in b/requirements/docker.in new file mode 100644 index 00000000..4906929c --- /dev/null +++ b/requirements/docker.in @@ -0,0 +1,3 @@ +-c base.txt + +gunicorn>=20.1.0 diff --git a/requirements/docker.txt b/requirements/docker.txt new file mode 100644 index 00000000..3a8ab334 --- /dev/null +++ b/requirements/docker.txt @@ -0,0 +1,12 @@ +# +# This file is autogenerated by pip-compile with Python 3.10 +# by the following command: +# +# pip-compile docker.in +# +gunicorn==21.2.0 + # via -r docker.in +packaging==23.2 + # via + # -c base.txt + # gunicorn