From 909c43be404f147a467ce7f169e5338523907093 Mon Sep 17 00:00:00 2001 From: Nils Wisiol Date: Fri, 30 Aug 2024 16:25:59 +0200 Subject: [PATCH] chore(GH actions): use docker compose instead of docker-compose This change is necessary as docker-compose is depricated since July 2023 and GitHub removed docker-compose from their action runner images on 1 Apr 2024. More info: https://github.com/actions/runner-images/issues/9557 --- .github/workflows/build/action.yml | 4 ++-- .github/workflows/test.yml | 14 +++++++------- README.md | 24 ++++++++++++------------ docker-compose.dev.yml | 2 -- docker-compose.test-api.yml | 2 -- docker-compose.test-e2e2.yml | 2 -- docker-compose.yml | 2 -- test/e2e2/README.md | 8 ++++---- 8 files changed, 25 insertions(+), 33 deletions(-) diff --git a/.github/workflows/build/action.yml b/.github/workflows/build/action.yml index b7df8d581..a5d28cd02 100644 --- a/.github/workflows/build/action.yml +++ b/.github/workflows/build/action.yml @@ -12,7 +12,7 @@ runs: uses: actions/checkout@v2 - name: Build Images shell: bash - run: docker-compose build ${{ inputs.images }} + run: docker compose build ${{ inputs.images }} - name: Build e2e2 Image shell: bash - run: docker-compose -f docker-compose.yml -f docker-compose.test-e2e2.yml build libfaketime && docker-compose -f docker-compose.yml -f docker-compose.test-e2e2.yml build test-e2e2 + run: docker compose -f docker-compose.yml -f docker-compose.test-e2e2.yml build libfaketime && docker compose -f docker-compose.yml -f docker-compose.test-e2e2.yml build test-e2e2 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7f469546b..f1e1084af 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -64,7 +64,7 @@ jobs: with: images: api dbapi nslord nsmaster dblord dbmaster - name: Check for missing migrations - run: docker-compose run -T api sh -c "./wait-dbapi && python manage.py makemigrations --check" + run: docker compose run -T api sh -c "./wait-dbapi && python manage.py makemigrations --check" test-e2e2: # runs e2e2 tests @@ -75,13 +75,13 @@ jobs: - name: Build images uses: ./.github/workflows/build - name: Run e2e2 Tests - run: docker-compose -f docker-compose.yml -f docker-compose.test-e2e2.yml run -T test-e2e2 sh -c "./apiwait 300 && python3 -m pytest -vv --skip-performance-tests ." + run: docker compose -f docker-compose.yml -f docker-compose.test-e2e2.yml run -T test-e2e2 sh -c "./apiwait 300 && python3 -m pytest -vv --skip-performance-tests ." - name: e2e2 Tests Logs and Cleanup if: always() run: | - docker-compose -f docker-compose.yml -f docker-compose.test-e2e2.yml ps + docker compose -f docker-compose.yml -f docker-compose.test-e2e2.yml ps grep 'desec/' /var/log/syslog - docker-compose -f docker-compose.yml -f docker-compose.test-e2e2.yml down -v + docker compose -f docker-compose.yml -f docker-compose.test-e2e2.yml down -v test-api: # runs API tests @@ -94,10 +94,10 @@ jobs: with: images: api dbapi nslord nsmaster dblord dbmaster - name: Run API Tests - run: docker-compose -f docker-compose.yml -f docker-compose.test-api.yml run -T api bash -c "./entrypoint-tests.sh" + run: docker compose -f docker-compose.yml -f docker-compose.test-api.yml run -T api bash -c "./entrypoint-tests.sh" - name: API Tests Logs and Cleanup if: always() run: | - docker-compose -f docker-compose.yml -f docker-compose.test-api.yml ps + docker compose -f docker-compose.yml -f docker-compose.test-api.yml ps grep 'desec/' /var/log/syslog - docker-compose -f docker-compose.yml -f docker-compose.test-api.yml down -v + docker compose -f docker-compose.yml -f docker-compose.test-api.yml down -v diff --git a/README.md b/README.md index 6078f223c..e20ce9b51 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ deSEC Stack =========== -This is a docker-compose application providing the basic stack for deSEC name services. It consists of +This is a docker compose application providing the basic stack for deSEC name services. It consists of - `nslord`: Eventually authoritative DNS server (PowerDNS). DNSSEC keying material is generated here. - `nsmaster`: Stealth authoritative DNS server (PowerDNS). Receives fully signed AXFR zone transfers from `nslord`. No access to keys. @@ -81,7 +81,7 @@ Development: Production: - $ docker-compose build && docker-compose up + $ docker compose build && docker compose up Storage ------- @@ -135,11 +135,11 @@ While there are certainly many ways to get started hacking desec-stack, here is 1. **Requirements.** This guide is intended and tested on Ubuntu 20.20. However, many other Linux distributions will also do fine. - For desec-stack, [docker](https://docs.docker.com/install/linux/docker-ce/ubuntu/) and [docker-compose](https://docs.docker.com/compose/install/) are required. + For desec-stack, [docker and docker compose v2](https://docs.docker.com/engine/install/ubuntu/) are required. Further tools that are required to start hacking are git and curl. Recommended, but not strictly required for desec-stack development is to use certbot along with Let's Encrypt and PyCharm. jq, httpie, libmariadbclient-dev, libpq-dev, python3-dev (>= 3.11) and python3-venv (>= 3.11) are useful if you want to follow this guide. - The webapp requires Node.js. To install everything you need for this guide except docker and docker-compose, use + The webapp requires Node.js. To install everything you need for this guide except docker and docker compose, use sudo apt install certbot curl git httpie jq libmariadbclient-dev libpq-dev nodejs npm python3-dev python3-venv libmemcached-dev @@ -224,7 +224,7 @@ While there are certainly many ways to get started hacking desec-stack, here is The last two steps need to be repeated whenever the certificates are renewed. While any location for the certificates is fine, the `certs/` folder is configured to be ignored by git so that private keys do not accidentally end up being committed. -1. **Configure desec-stack.** As docker-compose application, desec-stack is configured by environment variables defined in the `.env` file in the project root directory. +1. **Configure desec-stack.** As docker compose application, desec-stack is configured by environment variables defined in the `.env` file in the project root directory. Because it contains sensitive information for each deployment, `.env` is not part of the repository and ignored by git. However, we ship `.env.default` and `.env.dev` with templates for production and development, respectively. `.env.dev` is almost good enough for a basic development system, so let's use that as a basis: @@ -238,7 +238,7 @@ While there are certainly many ways to get started hacking desec-stack, here is Additionally, the VPN server for the replication network needs to be equipped with a pre-shared key (PSK) and a public key infrastructure (PKI). To generate the PSK, use the openvpn-server container: - docker-compose build openvpn-server && docker-compose run openvpn-server openvpn --genkey --secret /dev/stdout > openvpn-server/secrets/ta.key + docker compose build openvpn-server && docker compose run openvpn-server openvpn --genkey --secret /dev/stdout > openvpn-server/secrets/ta.key To build the PKI, we recommend [easy RSA](https://github.com/OpenVPN/easy-rsa). **Please note that PKI instructions here are for development deployments only!** @@ -291,7 +291,7 @@ While there are certainly many ways to get started hacking desec-stack, here is A convenient way to create a test user account is via - docker-compose exec api python3 manage.py shell -c 'from desecapi.models import User; User.objects.create_user(email="test@example.com", password="test1234");' + docker compose exec api python3 manage.py shell -c 'from desecapi.models import User; User.objects.create_user(email="test@example.com", password="test1234");' but users can also be created by signing up via the web GUI. The latter, however, requires that you can read email that is sent from your local setup. @@ -314,9 +314,9 @@ While there are certainly many ways to get started hacking desec-stack, here is to see if the nameserver is behaving as expected. -1. **(Optional) Configure PyCharm for API Development.** As a docker-compose application, desec-stack takes a while to start. +1. **(Optional) Configure PyCharm for API Development.** As a docker compose application, desec-stack takes a while to start. Additionally, it is hard to connect a debugger to the docker containers. - Our recommended solution is to develop the API using Django tests running outside the docker-compose application. + Our recommended solution is to develop the API using Django tests running outside the docker compose application. This will dramatically decrease the time required for running the Django tests and enable just-in-time debugging in PyCharm. Also, it will enable you to browse dependencies and code within PyCharm and thus ease debugging. @@ -351,7 +351,7 @@ While there are certainly many ways to get started hacking desec-stack, here is Fourth, run the database: - docker-compose -f docker-compose.yml -f docker-compose.test-api.yml up -d dbapi + docker compose -f docker-compose.yml -f docker-compose.test-api.yml up -d dbapi Finally, you can manage Django using the `manage.py` CLI. As an example, to run the tests, use @@ -369,8 +369,8 @@ While there are certainly many ways to get started hacking desec-stack, here is 3. Fill the Custom Settings field with the path to the `settings_quick_test` module. 4. At the bottom in the "Before launch" sections, add an "External tool" with the following settings: - Name: `Postgres Test Container` - - Program: `docker-compose` - - Arguments: `-f docker-compose.yml -f docker-compose.test-api.yml up -d dbapi` + - Program: `docker` + - Arguments: `compose -f docker-compose.yml -f docker-compose.test-api.yml up -d dbapi` 1. To see if the test configuration is working, right-click on the api folder in the project view and select Run Test. (Note that the first attempt may fail in case the `dbapi` container does not start up fast enough. In that case, just try again.) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 1c43d5312..d5b54e95b 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -1,5 +1,3 @@ -version: '2.3' - # mostly extending from main .yml services: www: diff --git a/docker-compose.test-api.yml b/docker-compose.test-api.yml index ca7494222..5de880b72 100644 --- a/docker-compose.test-api.yml +++ b/docker-compose.test-api.yml @@ -1,5 +1,3 @@ -version: '2.3' - # mostly extending from main .yml services: api: diff --git a/docker-compose.test-e2e2.yml b/docker-compose.test-e2e2.yml index 23273562d..6496a483f 100644 --- a/docker-compose.test-e2e2.yml +++ b/docker-compose.test-e2e2.yml @@ -1,5 +1,3 @@ -version: '2.3' - # mostly extending from main .yml services: www: diff --git a/docker-compose.yml b/docker-compose.yml index ac83c5b2a..d29426de9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: '2.3' - services: www: build: diff --git a/test/e2e2/README.md b/test/e2e2/README.md index 575ac9f75..c202bc6bc 100644 --- a/test/e2e2/README.md +++ b/test/e2e2/README.md @@ -6,14 +6,14 @@ A collection of tests against the stack written in python and pytest. The tests can be run from the **CLI** using - docker-compose -f docker-compose.yml -f docker-compose.test-e2e2.yml build libfaketime - docker-compose -f docker-compose.yml -f docker-compose.test-e2e2.yml build test-e2e2 - docker-compose -f docker-compose.yml -f docker-compose.test-e2e2.yml up test-e2e2 + docker compose -f docker-compose.yml -f docker-compose.test-e2e2.yml build libfaketime + docker compose -f docker-compose.yml -f docker-compose.test-e2e2.yml build test-e2e2 + docker compose -f docker-compose.yml -f docker-compose.test-e2e2.yml up test-e2e2 If you had any changes to other containers (say `api`), then also rebuild them. To run the test in **pycharm**, make sure that pytest is installed in the Python environment that pycharm is using. -Then add the docker-compose environment to your pycharm build configuration. Note that you need to update the pycharm +Then add the docker compose environment to your pycharm build configuration. Note that you need to update the pycharm environment configuration when you update the corresponding variables in your `.env`. When run from pycharm, the tests are not run inside a docker container and have no access to any self-signed