diff --git a/.github/workflows/build/action.yml b/.github/workflows/build/action.yml index b7df8d581..a5d28cd02 100644 --- a/.github/workflows/build/action.yml +++ b/.github/workflows/build/action.yml @@ -12,7 +12,7 @@ runs: uses: actions/checkout@v2 - name: Build Images shell: bash - run: docker-compose build ${{ inputs.images }} + run: docker compose build ${{ inputs.images }} - name: Build e2e2 Image shell: bash - run: docker-compose -f docker-compose.yml -f docker-compose.test-e2e2.yml build libfaketime && docker-compose -f docker-compose.yml -f docker-compose.test-e2e2.yml build test-e2e2 + run: docker compose -f docker-compose.yml -f docker-compose.test-e2e2.yml build libfaketime && docker compose -f docker-compose.yml -f docker-compose.test-e2e2.yml build test-e2e2 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7f469546b..f1e1084af 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -64,7 +64,7 @@ jobs: with: images: api dbapi nslord nsmaster dblord dbmaster - name: Check for missing migrations - run: docker-compose run -T api sh -c "./wait-dbapi && python manage.py makemigrations --check" + run: docker compose run -T api sh -c "./wait-dbapi && python manage.py makemigrations --check" test-e2e2: # runs e2e2 tests @@ -75,13 +75,13 @@ jobs: - name: Build images uses: ./.github/workflows/build - name: Run e2e2 Tests - run: docker-compose -f docker-compose.yml -f docker-compose.test-e2e2.yml run -T test-e2e2 sh -c "./apiwait 300 && python3 -m pytest -vv --skip-performance-tests ." + run: docker compose -f docker-compose.yml -f docker-compose.test-e2e2.yml run -T test-e2e2 sh -c "./apiwait 300 && python3 -m pytest -vv --skip-performance-tests ." - name: e2e2 Tests Logs and Cleanup if: always() run: | - docker-compose -f docker-compose.yml -f docker-compose.test-e2e2.yml ps + docker compose -f docker-compose.yml -f docker-compose.test-e2e2.yml ps grep 'desec/' /var/log/syslog - docker-compose -f docker-compose.yml -f docker-compose.test-e2e2.yml down -v + docker compose -f docker-compose.yml -f docker-compose.test-e2e2.yml down -v test-api: # runs API tests @@ -94,10 +94,10 @@ jobs: with: images: api dbapi nslord nsmaster dblord dbmaster - name: Run API Tests - run: docker-compose -f docker-compose.yml -f docker-compose.test-api.yml run -T api bash -c "./entrypoint-tests.sh" + run: docker compose -f docker-compose.yml -f docker-compose.test-api.yml run -T api bash -c "./entrypoint-tests.sh" - name: API Tests Logs and Cleanup if: always() run: | - docker-compose -f docker-compose.yml -f docker-compose.test-api.yml ps + docker compose -f docker-compose.yml -f docker-compose.test-api.yml ps grep 'desec/' /var/log/syslog - docker-compose -f docker-compose.yml -f docker-compose.test-api.yml down -v + docker compose -f docker-compose.yml -f docker-compose.test-api.yml down -v diff --git a/README.md b/README.md index 6078f223c..515af9d0d 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ deSEC Stack =========== -This is a docker-compose application providing the basic stack for deSEC name services. It consists of +This is a docker compose application providing the basic stack for deSEC name services. It consists of - `nslord`: Eventually authoritative DNS server (PowerDNS). DNSSEC keying material is generated here. - `nsmaster`: Stealth authoritative DNS server (PowerDNS). Receives fully signed AXFR zone transfers from `nslord`. No access to keys. @@ -81,7 +81,7 @@ Development: Production: - $ docker-compose build && docker-compose up + $ docker compose build && docker compose up Storage ------- @@ -133,13 +133,13 @@ Development: Getting Started Guide As desec-stack utilizes a number of different technologies and software packages, it requires some effort to setup a stack ready for development. While there are certainly many ways to get started hacking desec-stack, here is one way to do it. -1. **Requirements.** This guide is intended and tested on Ubuntu 20.20. +1. **Requirements.** This guide is intended and tested on Ubuntu 22.04 LTS. However, many other Linux distributions will also do fine. - For desec-stack, [docker](https://docs.docker.com/install/linux/docker-ce/ubuntu/) and [docker-compose](https://docs.docker.com/compose/install/) are required. + For desec-stack, [docker and docker compose v2](https://docs.docker.com/engine/install/ubuntu/) are required. Further tools that are required to start hacking are git and curl. Recommended, but not strictly required for desec-stack development is to use certbot along with Let's Encrypt and PyCharm. jq, httpie, libmariadbclient-dev, libpq-dev, python3-dev (>= 3.11) and python3-venv (>= 3.11) are useful if you want to follow this guide. - The webapp requires Node.js. To install everything you need for this guide except docker and docker-compose, use + The webapp requires Node.js. To install everything you need for this guide except docker and docker compose, use sudo apt install certbot curl git httpie jq libmariadbclient-dev libpq-dev nodejs npm python3-dev python3-venv libmemcached-dev @@ -224,7 +224,7 @@ While there are certainly many ways to get started hacking desec-stack, here is The last two steps need to be repeated whenever the certificates are renewed. While any location for the certificates is fine, the `certs/` folder is configured to be ignored by git so that private keys do not accidentally end up being committed. -1. **Configure desec-stack.** As docker-compose application, desec-stack is configured by environment variables defined in the `.env` file in the project root directory. +1. **Configure desec-stack.** As docker compose application, desec-stack is configured by environment variables defined in the `.env` file in the project root directory. Because it contains sensitive information for each deployment, `.env` is not part of the repository and ignored by git. However, we ship `.env.default` and `.env.dev` with templates for production and development, respectively. `.env.dev` is almost good enough for a basic development system, so let's use that as a basis: @@ -238,7 +238,7 @@ While there are certainly many ways to get started hacking desec-stack, here is Additionally, the VPN server for the replication network needs to be equipped with a pre-shared key (PSK) and a public key infrastructure (PKI). To generate the PSK, use the openvpn-server container: - docker-compose build openvpn-server && docker-compose run openvpn-server openvpn --genkey --secret /dev/stdout > openvpn-server/secrets/ta.key + docker compose build openvpn-server && docker compose run openvpn-server openvpn --genkey --secret /dev/stdout > openvpn-server/secrets/ta.key To build the PKI, we recommend [easy RSA](https://github.com/OpenVPN/easy-rsa). **Please note that PKI instructions here are for development deployments only!** @@ -291,7 +291,7 @@ While there are certainly many ways to get started hacking desec-stack, here is A convenient way to create a test user account is via - docker-compose exec api python3 manage.py shell -c 'from desecapi.models import User; User.objects.create_user(email="test@example.com", password="test1234");' + docker compose exec api python3 manage.py shell -c 'from desecapi.models import User; User.objects.create_user(email="test@example.com", password="test1234");' but users can also be created by signing up via the web GUI. The latter, however, requires that you can read email that is sent from your local setup. @@ -314,9 +314,9 @@ While there are certainly many ways to get started hacking desec-stack, here is to see if the nameserver is behaving as expected. -1. **(Optional) Configure PyCharm for API Development.** As a docker-compose application, desec-stack takes a while to start. +1. **(Optional) Configure PyCharm for API Development.** As a docker compose application, desec-stack takes a while to start. Additionally, it is hard to connect a debugger to the docker containers. - Our recommended solution is to develop the API using Django tests running outside the docker-compose application. + Our recommended solution is to develop the API using Django tests running outside the docker compose application. This will dramatically decrease the time required for running the Django tests and enable just-in-time debugging in PyCharm. Also, it will enable you to browse dependencies and code within PyCharm and thus ease debugging. @@ -351,7 +351,7 @@ While there are certainly many ways to get started hacking desec-stack, here is Fourth, run the database: - docker-compose -f docker-compose.yml -f docker-compose.test-api.yml up -d dbapi + docker compose -f docker-compose.yml -f docker-compose.test-api.yml up -d dbapi Finally, you can manage Django using the `manage.py` CLI. As an example, to run the tests, use @@ -369,8 +369,8 @@ While there are certainly many ways to get started hacking desec-stack, here is 3. Fill the Custom Settings field with the path to the `settings_quick_test` module. 4. At the bottom in the "Before launch" sections, add an "External tool" with the following settings: - Name: `Postgres Test Container` - - Program: `docker-compose` - - Arguments: `-f docker-compose.yml -f docker-compose.test-api.yml up -d dbapi` + - Program: `docker` + - Arguments: `compose -f docker-compose.yml -f docker-compose.test-api.yml up -d dbapi` 1. To see if the test configuration is working, right-click on the api folder in the project view and select Run Test. (Note that the first attempt may fail in case the `dbapi` container does not start up fast enough. In that case, just try again.) diff --git a/api/desecapi/migrations/0001_initial_squashed_again.py b/api/desecapi/migrations/0001_initial_squashed_again.py index dbb582bf8..4038d2d05 100644 --- a/api/desecapi/migrations/0001_initial_squashed_again.py +++ b/api/desecapi/migrations/0001_initial_squashed_again.py @@ -7,6 +7,7 @@ import django.db.models.deletion import re import uuid +from django.contrib.postgres.operations import CreateCollation class Migration(migrations.Migration): @@ -15,6 +16,13 @@ class Migration(migrations.Migration): dependencies = [] operations = [ + # Explanation: https://adamj.eu/tech/2023/02/23/migrate-django-postgresql-ci-fields-case-insensitive-collation/ + CreateCollation( + "case_insensitive", + provider="icu", + locale="und-u-ks-level2", + deterministic=False, + ), migrations.CreateModel( name="User", fields=[ @@ -37,7 +45,10 @@ class Migration(migrations.Migration): ( "email", models.EmailField( - max_length=191, unique=True, verbose_name="email address" + db_collation="case_insensitive", + max_length=254, + unique=True, + verbose_name="email address", ), ), ("is_active", models.BooleanField(default=True)), diff --git a/api/desecapi/migrations/0001_squashed_0031_alter_user_email.py b/api/desecapi/migrations/0001_squashed_0031_alter_user_email.py new file mode 100644 index 000000000..a8de890c3 --- /dev/null +++ b/api/desecapi/migrations/0001_squashed_0031_alter_user_email.py @@ -0,0 +1,1068 @@ +# Generated by Django 5.1 on 2024-08-30 13:27 + +import datetime +import desecapi.models.base +import desecapi.models.captcha +import desecapi.models.domains +import desecapi.models.donation +import desecapi.models.mfa +import desecapi.models.tokens +import desecapi.models.users +import django.contrib.postgres.constraints +import django.contrib.postgres.fields +import django.contrib.postgres.indexes +import django.contrib.postgres.operations +import django.core.validators +import django.db.migrations.operations.special +import django.db.models.deletion +import django.db.models.expressions +import netfields.fields +import re +import uuid +from django.conf import settings +from django.db import migrations, models +from django.db.models import F, Q + + +# Functions from the following migrations need manual copying. +# Move them and any dependencies into this file, then update the +# RunPython operations to refer to the local versions: +# desecapi.migrations.0007_email_citext +# desecapi.migrations.0019_alter_user_is_active +# desecapi.migrations.0020_user_email_verified +# desecapi.migrations.0025_alter_token_max_age_alter_token_max_unused_period +# desecapi.migrations.0027_user_credentials_changed +# desecapi.migrations.0031_alter_user_email + + +def alter_user_is_active_forward(apps, schema_editor): + User = apps.get_model("desecapi", "User") + db_alias = schema_editor.connection.alias + User.objects.using(db_alias).filter( + is_active=False, last_login__isnull=True + ).update(is_active=None) + + +def alter_user_is_active_reverse(apps, schema_editor): + User = apps.get_model("desecapi", "User") + db_alias = schema_editor.connection.alias + User.objects.using(db_alias).filter(is_active__isnull=True).update(is_active=False) + + +def user_email_verified_forward(apps, schema_editor): + User = apps.get_model("desecapi", "User") + db_alias = schema_editor.connection.alias + User.objects.using(db_alias).filter( + Q(is_active=True) | Q(last_login__isnull=False), + created__date__gte=datetime.date(2019, 11, 1), + ).update(email_verified=F("created")) + + +def alter_token_max_age_alter_token_max_unused_period_forward(apps, schema_editor): + max_interval = datetime.timedelta(days=365000) + Token = apps.get_model("desecapi", "Token") + db_alias = schema_editor.connection.alias + Token.objects.using(db_alias).filter(max_age__gt=max_interval).update( + max_age=max_interval + ) + Token.objects.using(db_alias).filter(max_unused_period__gt=max_interval).update( + max_unused_period=max_interval + ) + + +def user_credentials_changed_forward(apps, schema_editor): + User = apps.get_model("desecapi", "User") + db_alias = schema_editor.connection.alias + User.objects.using(db_alias).update(credentials_changed=F("created")) + + +class Migration(migrations.Migration): + + replaces = [ + ("desecapi", "0001_initial_squashed_again"), + ("desecapi", "0002_unmanaged_donations"), + ("desecapi", "0003_rr_content"), + ("desecapi", "0004_immortal_domains"), + ("desecapi", "0005_subname_validation"), + ("desecapi", "0006_cname_exclusivity"), + ("desecapi", "0007_email_citext"), + ("desecapi", "0008_token_perm_manage_tokens"), + ("desecapi", "0009_token_allowed_subnets"), + ("desecapi", "0010_token_expiration"), + ("desecapi", "0011_captcha_kind"), + ("desecapi", "0012_rrset_label_length"), + ("desecapi", "0013_user_needs_captcha"), + ("desecapi", "0014_replication"), + ("desecapi", "0015_rrset_touched_index"), + ("desecapi", "0016_default_auto_field"), + ("desecapi", "0017_alter_user_limit_domains"), + ("desecapi", "0018_tokendomainpolicy"), + ("desecapi", "0019_alter_user_is_active"), + ("desecapi", "0020_user_email_verified"), + ("desecapi", "0021_authenticatednoopuseraction"), + ("desecapi", "0022_user_outreach_preference"), + ("desecapi", "0023_authenticatedemailuseraction"), + ("desecapi", "0024_authenticatedchangeoutreachpreferenceuseraction"), + ("desecapi", "0025_alter_token_max_age_alter_token_max_unused_period"), + ("desecapi", "0026_remove_domain_replicated_and_more"), + ("desecapi", "0027_user_credentials_changed"), + ( + "desecapi", + "0028_authenticatedcreatetotpfactoruseraction_basefactor_and_more", + ), + ("desecapi", "0029_token_mfa"), + ("desecapi", "0030_blockedsubnet_blockedsubnet_subnet_idx"), + ("desecapi", "0031_alter_user_email"), + ] + + initial = True + + dependencies = [] + + operations = [ + django.contrib.postgres.operations.CreateCollation( + name="case_insensitive", + locale="und-u-ks-level2", + provider="icu", + deterministic=False, + ), + migrations.CreateModel( + name="User", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("password", models.CharField(max_length=128, verbose_name="password")), + ( + "last_login", + models.DateTimeField( + blank=True, null=True, verbose_name="last login" + ), + ), + ( + "email", + models.EmailField( + db_collation="case_insensitive", + max_length=254, + unique=True, + verbose_name="email address", + ), + ), + ("is_active", models.BooleanField(default=True)), + ("is_admin", models.BooleanField(default=False)), + ("created", models.DateTimeField(auto_now_add=True)), + ( + "limit_domains", + models.IntegerField( + blank=True, + default=desecapi.models.users.User._limit_domains_default, + null=True, + ), + ), + ], + options={ + "abstract": False, + }, + ), + migrations.CreateModel( + name="Domain", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("created", models.DateTimeField(auto_now_add=True)), + ( + "name", + models.CharField( + max_length=191, + unique=True, + validators=[ + desecapi.models.base.validate_lower, + django.core.validators.RegexValidator( + code="invalid_domain_name", + flags=re.RegexFlag["IGNORECASE"], + message="Domain names must be labels separated by dots. Labels may consist of up to 63 letters, digits, hyphens, and underscores. The last label may not contain an underscore.", + regex="^(([a-z0-9_-]{1,63})\\.)*[a-z0-9-]{1,63}$", + ), + ], + ), + ), + ( + "owner", + models.ForeignKey( + on_delete=django.db.models.deletion.PROTECT, + related_name="domains", + to=settings.AUTH_USER_MODEL, + ), + ), + ("published", models.DateTimeField(blank=True, null=True)), + ( + "minimum_ttl", + models.PositiveIntegerField( + default=desecapi.models.domains.Domain._minimum_ttl_default + ), + ), + ("renewal_changed", models.DateTimeField(auto_now_add=True)), + ( + "renewal_state", + models.IntegerField( + choices=[ + (0, "Immortal"), + (1, "Fresh"), + (2, "Notified"), + (3, "Warned"), + ], + default=0, + ), + ), + ], + options={ + "ordering": ("created",), + }, + ), + migrations.CreateModel( + name="RRset", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("created", models.DateTimeField(auto_now_add=True)), + ("touched", models.DateTimeField(auto_now=True)), + ( + "subname", + models.CharField( + blank=True, + max_length=178, + validators=[ + desecapi.models.base.validate_lower, + django.core.validators.RegexValidator( + code="invalid_subname", + message="Subname can only use (lowercase) a-z, 0-9, ., -, and _, may start with a '*.', or just be '*'.", + regex="^([*]|(([*][.])?([a-z0-9_-]+[.])*[a-z0-9_-]+))$", + ), + ], + ), + ), + ( + "type", + models.CharField( + max_length=10, + validators=[ + desecapi.models.base.validate_upper, + django.core.validators.RegexValidator( + code="invalid_type", + message="Type must be uppercase alphanumeric and start with a letter.", + regex="^[A-Z][A-Z0-9]*$", + ), + ], + ), + ), + ("ttl", models.PositiveIntegerField()), + ( + "domain", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to="desecapi.domain", + ), + ), + ], + options={ + "unique_together": {("domain", "subname", "type")}, + }, + ), + migrations.CreateModel( + name="AuthenticatedAction", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ], + options={ + "managed": False, + }, + ), + migrations.CreateModel( + name="AuthenticatedUserAction", + fields=[ + ( + "authenticatedaction_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="desecapi.authenticatedaction", + ), + ), + ], + options={ + "managed": False, + }, + bases=("desecapi.authenticatedaction",), + ), + migrations.CreateModel( + name="AuthenticatedDeleteUserAction", + fields=[ + ( + "authenticateduseraction_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="desecapi.authenticateduseraction", + ), + ), + ], + options={ + "managed": False, + }, + bases=("desecapi.authenticateduseraction",), + ), + migrations.CreateModel( + name="AuthenticatedResetPasswordUserAction", + fields=[ + ( + "authenticateduseraction_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="desecapi.authenticateduseraction", + ), + ), + ("new_password", models.CharField(max_length=128)), + ], + options={ + "managed": False, + }, + bases=("desecapi.authenticateduseraction",), + ), + migrations.CreateModel( + name="Captcha", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("created", models.DateTimeField(auto_now_add=True)), + ( + "content", + models.CharField( + default=desecapi.models.captcha.captcha_default_content, + max_length=24, + ), + ), + ], + ), + migrations.CreateModel( + name="Token", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ( + "created", + models.DateTimeField(auto_now_add=True, verbose_name="Created"), + ), + ( + "key", + models.CharField( + db_index=True, max_length=128, unique=True, verbose_name="Key" + ), + ), + ( + "name", + models.CharField(blank=True, max_length=64, verbose_name="Name"), + ), + ( + "user", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="auth_tokens", + to=settings.AUTH_USER_MODEL, + verbose_name="User", + ), + ), + ("last_used", models.DateTimeField(blank=True, null=True)), + ], + options={ + "verbose_name": "Token", + "verbose_name_plural": "Tokens", + }, + ), + migrations.CreateModel( + name="AuthenticatedActivateUserAction", + fields=[ + ( + "authenticateduseraction_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="desecapi.authenticateduseraction", + ), + ), + ("domain", models.CharField(max_length=191)), + ], + options={ + "managed": False, + }, + bases=("desecapi.authenticateduseraction",), + ), + migrations.CreateModel( + name="AuthenticatedChangeEmailUserAction", + fields=[ + ( + "authenticateduseraction_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="desecapi.authenticateduseraction", + ), + ), + ("new_email", models.EmailField(max_length=254)), + ], + options={ + "managed": False, + }, + bases=("desecapi.authenticateduseraction",), + ), + migrations.CreateModel( + name="AuthenticatedBasicUserAction", + fields=[ + ( + "authenticatedaction_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="desecapi.authenticatedaction", + ), + ), + ], + options={ + "managed": False, + }, + bases=("desecapi.authenticatedaction",), + ), + migrations.CreateModel( + name="AuthenticatedDomainBasicUserAction", + fields=[ + ( + "authenticatedbasicuseraction_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="desecapi.authenticatedbasicuseraction", + ), + ), + ], + options={ + "managed": False, + }, + bases=("desecapi.authenticatedbasicuseraction",), + ), + migrations.CreateModel( + name="AuthenticatedRenewDomainBasicUserAction", + fields=[ + ( + "authenticateddomainbasicuseraction_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="desecapi.authenticateddomainbasicuseraction", + ), + ), + ], + options={ + "managed": False, + }, + bases=("desecapi.authenticateddomainbasicuseraction",), + ), + migrations.CreateModel( + name="Donation", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "created", + models.DateTimeField( + default=desecapi.models.donation.Donation._created_default + ), + ), + ("name", models.CharField(max_length=255)), + ("iban", models.CharField(max_length=34)), + ("bic", models.CharField(max_length=11)), + ("amount", models.DecimalField(decimal_places=2, max_digits=8)), + ("message", models.CharField(blank=True, max_length=255)), + ( + "due", + models.DateTimeField( + default=desecapi.models.donation.Donation._due_default + ), + ), + ( + "mref", + models.CharField( + default=desecapi.models.donation.Donation._mref_default, + max_length=32, + ), + ), + ("email", models.EmailField(blank=True, max_length=255)), + ], + options={ + "managed": False, + }, + ), + migrations.CreateModel( + name="RR", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("created", models.DateTimeField(auto_now_add=True)), + ("content", models.TextField()), + ( + "rrset", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="records", + to="desecapi.rrset", + ), + ), + ], + ), + django.contrib.postgres.operations.BtreeGistExtension(), + migrations.AddConstraint( + model_name="rrset", + constraint=django.contrib.postgres.constraints.ExclusionConstraint( + expressions=[ + ("domain", "="), + ("subname", "="), + ( + django.db.models.expressions.RawSQL("int4(type = 'CNAME')", ()), + "<>", + ), + ], + name="cname_exclusivity", + ), + ), + migrations.AddField( + model_name="token", + name="perm_manage_tokens", + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name="token", + name="allowed_subnets", + field=django.contrib.postgres.fields.ArrayField( + base_field=netfields.fields.CidrAddressField(max_length=43), + default=desecapi.models.tokens.Token._allowed_subnets_default, + size=None, + ), + ), + migrations.AddField( + model_name="token", + name="max_age", + field=models.DurationField( + default=None, + null=True, + validators=[ + django.core.validators.MinValueValidator(datetime.timedelta(0)) + ], + ), + ), + migrations.AddField( + model_name="token", + name="max_unused_period", + field=models.DurationField( + default=None, + null=True, + validators=[ + django.core.validators.MinValueValidator(datetime.timedelta(0)) + ], + ), + ), + migrations.AddField( + model_name="captcha", + name="kind", + field=models.CharField( + choices=[("image", "Image"), ("audio", "Audio")], + default="image", + max_length=24, + ), + ), + migrations.AlterField( + model_name="captcha", + name="content", + field=models.CharField(default="", max_length=24), + ), + migrations.AlterField( + model_name="rrset", + name="subname", + field=models.CharField( + blank=True, + max_length=178, + validators=[ + desecapi.models.base.validate_lower, + django.core.validators.RegexValidator( + code="invalid_subname", + message="Subname can only use (lowercase) a-z, 0-9, ., -, and _, may start with a '*.', or just be '*'. Components may not exceed 63 characters.", + regex="^([*]|(([*][.])?([a-z0-9_-]{1,63}[.])*[a-z0-9_-]{1,63}))$", + ), + ], + ), + ), + migrations.AddField( + model_name="user", + name="needs_captcha", + field=models.BooleanField(default=True), + ), + migrations.AddField( + model_name="domain", + name="replicated", + field=models.DateTimeField(blank=True, null=True), + ), + migrations.AddField( + model_name="domain", + name="replication_duration", + field=models.DurationField(blank=True, null=True), + ), + migrations.AlterField( + model_name="rrset", + name="touched", + field=models.DateTimeField(auto_now=True, db_index=True), + ), + migrations.AlterField( + model_name="domain", + name="id", + field=models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + migrations.AlterField( + model_name="rr", + name="id", + field=models.BigAutoField( + auto_created=True, primary_key=True, serialize=False, verbose_name="ID" + ), + ), + migrations.AlterField( + model_name="user", + name="limit_domains", + field=models.PositiveIntegerField( + blank=True, + default=desecapi.models.users.User._limit_domains_default, + null=True, + ), + ), + migrations.AlterField( + model_name="token", + name="user", + field=models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL + ), + ), + migrations.CreateModel( + name="TokenDomainPolicy", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("perm_dyndns", models.BooleanField(default=False)), + ("perm_rrsets", models.BooleanField(default=False)), + ( + "domain", + models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="desecapi.domain", + ), + ), + ( + "token", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, to="desecapi.token" + ), + ), + ( + "token_user", + models.ForeignKey( + db_constraint=False, + on_delete=django.db.models.deletion.CASCADE, + related_name="+", + to=settings.AUTH_USER_MODEL, + ), + ), + ], + ), + migrations.AddField( + model_name="token", + name="domain_policies", + field=models.ManyToManyField( + through="desecapi.TokenDomainPolicy", to="desecapi.domain" + ), + ), + migrations.AddConstraint( + model_name="tokendomainpolicy", + constraint=models.UniqueConstraint( + fields=("token", "domain"), name="unique_entry" + ), + ), + migrations.AddConstraint( + model_name="tokendomainpolicy", + constraint=models.UniqueConstraint( + condition=models.Q(("domain__isnull", True)), + fields=("token",), + name="unique_entry_null_domain", + ), + ), + migrations.AlterModelOptions( + name="token", + options={}, + ), + migrations.AddConstraint( + model_name="token", + constraint=models.UniqueConstraint( + fields=("id", "user"), name="unique_id_user" + ), + ), + migrations.AddConstraint( + model_name="domain", + constraint=models.UniqueConstraint( + fields=("id", "owner"), name="unique_id_owner" + ), + ), + migrations.RunSQL( + sql="ALTER TABLE desecapi_tokendomainpolicy ADD FOREIGN KEY ( domain_id, token_user_id ) REFERENCES desecapi_domain ( id, owner_id ), ADD FOREIGN KEY ( token_id, token_user_id ) REFERENCES desecapi_token ( id, user_id );", + reverse_sql="", + ), + migrations.AlterField( + model_name="user", + name="is_active", + field=models.BooleanField(default=True, null=True), + ), + migrations.RunPython( + code=alter_user_is_active_forward, + reverse_code=alter_user_is_active_reverse, + ), + migrations.AddField( + model_name="user", + name="email_verified", + field=models.DateTimeField(blank=True, null=True), + ), + migrations.RunPython( + code=user_email_verified_forward, + reverse_code=django.db.migrations.operations.special.RunPython.noop, + ), + migrations.CreateModel( + name="AuthenticatedNoopUserAction", + fields=[ + ( + "authenticateduseraction_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="desecapi.authenticateduseraction", + ), + ), + ], + options={ + "managed": False, + }, + bases=("desecapi.authenticateduseraction",), + ), + migrations.AddField( + model_name="user", + name="outreach_preference", + field=models.BooleanField(default=True), + ), + migrations.CreateModel( + name="AuthenticatedEmailUserAction", + fields=[ + ( + "authenticatedbasicuseraction_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="desecapi.authenticatedbasicuseraction", + ), + ), + ], + options={ + "managed": False, + }, + bases=("desecapi.authenticatedbasicuseraction",), + ), + migrations.CreateModel( + name="AuthenticatedChangeOutreachPreferenceUserAction", + fields=[ + ( + "authenticatedemailuseraction_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="desecapi.authenticatedemailuseraction", + ), + ), + ("outreach_preference", models.BooleanField()), + ], + options={ + "managed": False, + }, + bases=("desecapi.authenticatedemailuseraction",), + ), + migrations.AlterField( + model_name="token", + name="max_age", + field=models.DurationField( + default=None, + null=True, + validators=[ + django.core.validators.MinValueValidator(datetime.timedelta(0)), + django.core.validators.MaxValueValidator( + datetime.timedelta(days=365000) + ), + ], + ), + ), + migrations.AlterField( + model_name="token", + name="max_unused_period", + field=models.DurationField( + default=None, + null=True, + validators=[ + django.core.validators.MinValueValidator(datetime.timedelta(0)), + django.core.validators.MaxValueValidator( + datetime.timedelta(days=365000) + ), + ], + ), + ), + migrations.RunPython( + code=alter_token_max_age_alter_token_max_unused_period_forward, + reverse_code=django.db.migrations.operations.special.RunPython.noop, + ), + migrations.RemoveField( + model_name="domain", + name="replicated", + ), + migrations.RemoveField( + model_name="domain", + name="replication_duration", + ), + migrations.AddField( + model_name="user", + name="credentials_changed", + field=models.DateTimeField(auto_now_add=True, null=True), + ), + migrations.RunPython( + code=user_credentials_changed_forward, + reverse_code=django.db.migrations.operations.special.RunPython.noop, + ), + migrations.AlterField( + model_name="user", + name="credentials_changed", + field=models.DateTimeField(auto_now_add=True), + ), + migrations.CreateModel( + name="AuthenticatedCreateTOTPFactorUserAction", + fields=[ + ( + "authenticateduseraction_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="desecapi.authenticateduseraction", + ), + ), + ("name", models.CharField(blank=True, max_length=64)), + ], + options={ + "managed": False, + }, + bases=("desecapi.authenticateduseraction",), + ), + migrations.CreateModel( + name="BaseFactor", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + primary_key=True, + serialize=False, + ), + ), + ("created", models.DateTimeField(auto_now_add=True)), + ("last_used", models.DateTimeField(blank=True, null=True)), + ("name", models.CharField(blank=True, default="", max_length=64)), + ( + "user", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + to=settings.AUTH_USER_MODEL, + ), + ), + ], + ), + migrations.CreateModel( + name="TOTPFactor", + fields=[ + ( + "basefactor_ptr", + models.OneToOneField( + auto_created=True, + on_delete=django.db.models.deletion.CASCADE, + parent_link=True, + primary_key=True, + serialize=False, + to="desecapi.basefactor", + ), + ), + ( + "secret", + models.BinaryField( + default=desecapi.models.mfa.TOTPFactor._secret_default, + max_length=32, + ), + ), + ("last_verified_timestep", models.PositiveIntegerField(default=0)), + ], + bases=("desecapi.basefactor",), + ), + migrations.AddConstraint( + model_name="basefactor", + constraint=models.UniqueConstraint( + fields=("user", "name"), name="unique_user_name" + ), + ), + migrations.AddField( + model_name="token", + name="mfa", + field=models.BooleanField(default=None, null=True), + ), + migrations.CreateModel( + name="BlockedSubnet", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("created", models.DateTimeField(auto_now_add=True)), + ( + "asn", + models.PositiveBigIntegerField( + validators=[ + django.core.validators.MaxValueValidator(4294967295) + ] + ), + ), + ( + "subnet", + netfields.fields.CidrAddressField(max_length=43, unique=True), + ), + ("country", models.TextField()), + ("registry", models.TextField()), + ("allocation_date", models.DateField()), + ], + options={ + "indexes": [ + django.contrib.postgres.indexes.GistIndex( + fields=["subnet"], name="subnet_idx", opclasses=("inet_ops",) + ) + ], + }, + ), + ] diff --git a/api/desecapi/migrations/0007_email_citext.py b/api/desecapi/migrations/0007_email_citext.py index aac2b23fe..dc4c26ba1 100644 --- a/api/desecapi/migrations/0007_email_citext.py +++ b/api/desecapi/migrations/0007_email_citext.py @@ -1,7 +1,7 @@ # Generated by Django 3.1 on 2020-09-29 14:28 -import django.contrib.postgres.fields.citext -from django.contrib.postgres.operations import CITextExtension +# import django.contrib.postgres.fields.citext +# from django.contrib.postgres.operations import CITextExtension from django.db import migrations @@ -11,12 +11,12 @@ class Migration(migrations.Migration): ] operations = [ - CITextExtension(), - migrations.AlterField( - model_name="user", - name="email", - field=django.contrib.postgres.fields.citext.CIEmailField( - max_length=254, unique=True, verbose_name="email address" - ), - ), + # CITextExtension(), + # migrations.AlterField( + # model_name="user", + # name="email", + # field=django.contrib.postgres.fields.citext.CIEmailField( + # max_length=254, unique=True, verbose_name="email address" + # ), + # ), ] diff --git a/api/desecapi/migrations/0031_alter_user_email.py b/api/desecapi/migrations/0031_alter_user_email.py index e4ba9e896..5ddd31239 100644 --- a/api/desecapi/migrations/0031_alter_user_email.py +++ b/api/desecapi/migrations/0031_alter_user_email.py @@ -1,7 +1,7 @@ # Generated by Django 4.1.9 on 2023-06-08 16:40 -from django.contrib.postgres.operations import CreateCollation -from django.db import migrations, models +# from django.contrib.postgres.operations import CreateCollation +from django.db import migrations # , models class Migration(migrations.Migration): @@ -10,23 +10,23 @@ class Migration(migrations.Migration): ] operations = [ - # Explanation: https://adamj.eu/tech/2023/02/23/migrate-django-postgresql-ci-fields-case-insensitive-collation/ - CreateCollation( - "case_insensitive", - provider="icu", - locale="und-u-ks-level2", - deterministic=False, - ), - migrations.AlterField( - model_name="user", - name="email", - field=models.EmailField( - db_collation="case_insensitive", - max_length=254, - unique=True, - verbose_name="email address", - ), - ), + # # Explanation: https://adamj.eu/tech/2023/02/23/migrate-django-postgresql-ci-fields-case-insensitive-collation/ + # CreateCollation( + # "case_insensitive", + # provider="icu", + # locale="und-u-ks-level2", + # deterministic=False, + # ), + # migrations.AlterField( + # model_name="user", + # name="email", + # field=models.EmailField( + # db_collation="case_insensitive", + # max_length=254, + # unique=True, + # verbose_name="email address", + # ), + # ), migrations.RunSQL( sql='DROP EXTENSION IF EXISTS "citext"', reverse_sql='CREATE EXTENSION IF NOT EXISTS "citext"', diff --git a/api/desecapi/tests/test_token_policies.py b/api/desecapi/tests/test_token_policies.py index 5b91a1143..ba318fb07 100644 --- a/api/desecapi/tests/test_token_policies.py +++ b/api/desecapi/tests/test_token_policies.py @@ -22,6 +22,7 @@ def test_policies(self): kwargs.update(HTTP_AUTHORIZATION=f"Token {self.token_manage.plain}") response = self.client.get(url, **kwargs) self.assertStatus(response, status.HTTP_200_OK) + self.assertIn("rrsets", response.data) kwargs.update(HTTP_AUTHORIZATION=f"Token {self.token.plain}") response = self.client.get(url, **kwargs) diff --git a/api/desecapi/views/tokens.py b/api/desecapi/views/tokens.py index beaa74caa..f0c6159f3 100644 --- a/api/desecapi/views/tokens.py +++ b/api/desecapi/views/tokens.py @@ -54,7 +54,7 @@ def get(self, request, *args, **kwargs): self.get_object() # raises if token does not exist return Response( { - "domain": reverse( + "rrsets": reverse( "token_domain_policies-list", request=request, kwargs=kwargs ) } diff --git a/api/requirements.txt b/api/requirements.txt index 53d59b366..d2af9d2d1 100644 --- a/api/requirements.txt +++ b/api/requirements.txt @@ -1,8 +1,8 @@ -captcha~=0.5.0 +captcha~=0.6.0 celery~=5.4.0 -coverage~=7.5.4 -cryptography~=42.0.8 -Django~=5.0.6 +coverage~=7.6.0 +cryptography~=43.0.0 +Django~=5.1.0 django-cors-headers~=4.4.0 djangorestframework~=3.14.0 django-celery-email~=3.0.0 @@ -12,7 +12,7 @@ django-prometheus~=2.3.1 dnspython~=2.6.1 httpretty~=1.0.5 # 1.1 breaks tests. Does not run in production, so stick to it. pyotp~=2.9.0 -psycopg~=3.1.19 +psycopg~=3.2.1 psl-dns~=1.1.0 pylibmc~=1.6.3 pyyaml~=6.0.1 diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 1c43d5312..d5b54e95b 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -1,5 +1,3 @@ -version: '2.3' - # mostly extending from main .yml services: www: diff --git a/docker-compose.test-api.yml b/docker-compose.test-api.yml index ca7494222..5de880b72 100644 --- a/docker-compose.test-api.yml +++ b/docker-compose.test-api.yml @@ -1,5 +1,3 @@ -version: '2.3' - # mostly extending from main .yml services: api: diff --git a/docker-compose.test-e2e2.yml b/docker-compose.test-e2e2.yml index 23273562d..6496a483f 100644 --- a/docker-compose.test-e2e2.yml +++ b/docker-compose.test-e2e2.yml @@ -1,5 +1,3 @@ -version: '2.3' - # mostly extending from main .yml services: www: diff --git a/docker-compose.yml b/docker-compose.yml index ac83c5b2a..d29426de9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: '2.3' - services: www: build: diff --git a/test/e2e2/Dockerfile b/test/e2e2/Dockerfile index 9e0d7f924..b0d49122a 100644 --- a/test/e2e2/Dockerfile +++ b/test/e2e2/Dockerfile @@ -1,8 +1,16 @@ +FROM alpine/git +RUN apk -U add build-base \ + && git clone https://github.com/wolfcw/libfaketime /libfaketime +WORKDIR /libfaketime +RUN git checkout ba9ed5b2898f234cfcefbe5c694b7d89dcec4334 \ + && make \ + && make install + FROM python:3.11-alpine RUN apk add --no-cache bash curl -COPY --from=desec-stack_libfaketime /faketime.so /lib/libfaketime.so +COPY --from=0 /usr/local/lib/faketime/libfaketimeMT.so.1 /lib/libfaketime.so RUN mkdir -p /etc/faketime RUN mkdir /e2e diff --git a/test/e2e2/README.md b/test/e2e2/README.md index 575ac9f75..c202bc6bc 100644 --- a/test/e2e2/README.md +++ b/test/e2e2/README.md @@ -6,14 +6,14 @@ A collection of tests against the stack written in python and pytest. The tests can be run from the **CLI** using - docker-compose -f docker-compose.yml -f docker-compose.test-e2e2.yml build libfaketime - docker-compose -f docker-compose.yml -f docker-compose.test-e2e2.yml build test-e2e2 - docker-compose -f docker-compose.yml -f docker-compose.test-e2e2.yml up test-e2e2 + docker compose -f docker-compose.yml -f docker-compose.test-e2e2.yml build libfaketime + docker compose -f docker-compose.yml -f docker-compose.test-e2e2.yml build test-e2e2 + docker compose -f docker-compose.yml -f docker-compose.test-e2e2.yml up test-e2e2 If you had any changes to other containers (say `api`), then also rebuild them. To run the test in **pycharm**, make sure that pytest is installed in the Python environment that pycharm is using. -Then add the docker-compose environment to your pycharm build configuration. Note that you need to update the pycharm +Then add the docker compose environment to your pycharm build configuration. Note that you need to update the pycharm environment configuration when you update the corresponding variables in your `.env`. When run from pycharm, the tests are not run inside a docker container and have no access to any self-signed