diff --git a/.env.dist b/.env.dist index d2886e2..323f21f 100644 --- a/.env.dist +++ b/.env.dist @@ -34,6 +34,9 @@ ENGINE_WORKER_TIMEOUT=3600 ENGINE_WORKER_RETRY=4500 ENGINE_WORKER_MAX_ATTEMPTS=0 +# Django Q SQS Broker +SQS_BROKER_QUEUE_NAME=composite-handler-queue + # Run migrations MIGRATE=True diff --git a/README.md b/README.md index c81ab80..f635ad9 100644 --- a/README.md +++ b/README.md @@ -91,9 +91,18 @@ The following list of environment variables are supported: | `MIGRATE` | None | API, Engine | If "True" will run migrations + createcachetable on startup if entrypoint used. | | `INIT_SUPERUSER` | None | API, Engine | If "True" will attempt to create superuser. Needs standard Django envvars to be set (e.g. `DJANGO_SUPERUSER_USERNAME`, `DJANGO_SUPERUSER_EMAIL`, `DJANGO_SUPERUSER_PASSWORD`) if entrypoint used. | | `GUNICORN_WORKERS` | `2` | API | The value of [`--workers`](https://docs.gunicorn.org/en/stable/run.html) arg when running gunicorn | +| `SQS_BROKER_QUEUE_NAME` | None | API, Engine | If set, django-q [SQS broker](https://django-q.readthedocs.io/en/latest/brokers.html#amazon-sqs) will be used. Queue created if doesn't exist. If empty default [Django ORM broker](https://django-q.readthedocs.io/en/latest/brokers.html#django-orm) is used | Note that in order to access the S3 bucket, the Composite Handler assumes that valid AWS credentials are available in the environment - this can be in the former of [environment variables](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html), or in the form of ambient credentials. +### Django Q Broker + +By default Django Q will use the default [Django ORM](https://django-q.readthedocs.io/en/latest/brokers.html#django-orm) broker. + +The [SQS broker](https://django-q.readthedocs.io/en/latest/brokers.html#amazon-sqs) can be configured by specifying the `SQS_BROKER_QUEUE_NAME` environment variable. Default SQS broker behaviour is to create this queue if it is not found. + +As with S3, above, Composite Handler assumes that valid AWS credentials are available in the environment. + ## Building The project ships with a [`Dockerfile`](./Dockerfile): diff --git a/docker-compose.yml b/docker-compose.yml index fe7918c..247f757 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,6 +15,8 @@ services: - INIT_SUPERUSER=True ports: - "8000:8000" + volumes: + - $HOME/.aws:/srv/dlcs/.aws:ro engine: build: . command: /srv/dlcs/entrypoint-worker.sh diff --git a/src/app/settings.py b/src/app/settings.py index af2ff3d..58f1c28 100644 --- a/src/app/settings.py +++ b/src/app/settings.py @@ -156,9 +156,14 @@ "timeout": env("ENGINE_WORKER_TIMEOUT", cast=int, default=3600), "retry": env("ENGINE_WORKER_RETRY", cast=int, default=4500), "max_attempts": env("ENGINE_WORKER_MAX_ATTEMPTS", cast=int, default=0), - "orm": "default", } +if sqs_queue_name := env.str("SQS_BROKER_QUEUE_NAME", default=""): + Q_CLUSTER["name"] = sqs_queue_name + Q_CLUSTER["sqs"] = {} +else: + Q_CLUSTER["orm"] = "default" + SCRATCH_DIRECTORY = env.path("SCRATCH_DIRECTORY", default="/tmp/scratch") WEB_SERVER = {