diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 115935b0..eb8a4521 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,6 +1,19 @@ name: CI -"on": [push] +'on': + push: + branches-ignore: + # These should always correspond to pull requests, so ignore them for + # the push trigger and let them be triggered by the pull_request + # trigger, avoiding running the workflow twice. This is a minor + # optimization so there's no need to ensure this is comprehensive. + - 'dependabot/**' + - 'renovate/**' + - 'tickets/**' + - 'u/**' + tags: + - '*' + pull_request: {} jobs: lint: @@ -60,11 +73,11 @@ jobs: LTD_KEEPER_TEST_AWS_ID: ${{ secrets.LTD_KEEPER_TEST_AWS_ID }} LTD_KEEPER_TEST_AWS_SECRET: ${{ secrets.LTD_KEEPER_TEST_AWS_SECRET }} LTD_KEEPER_TEST_BUCKET: ${{ secrets.LTD_KEEPER_TEST_BUCKET }} - run: tox -e typing,${{matrix.db}},coverage-report # run tox using Python in path + run: tox -e typing,${{matrix.db}},coverage-report # run tox using Python in path - name: Run tox without external services if: ${{ !(matrix.python != 3.9 && matrix.db != 'postgres') }} - run: tox -e typing,${{matrix.db}},coverage-report # run tox using Python in path + run: tox -e typing,${{matrix.db}},coverage-report # run tox using Python in path docs: runs-on: ubuntu-latest @@ -72,6 +85,8 @@ jobs: steps: - uses: actions/checkout@v2 + with: + fetch-depth: 0 - name: Set up Python uses: actions/setup-python@v2 @@ -106,15 +121,18 @@ jobs: needs: [test] # Only do Docker builds of ticket branches and tagged releases, as well - # as JSick Codes branches. - if: startsWith(github.ref, 'refs/tags/') || startsWith(github.ref, 'refs/heads/tickets/') || startsWith(github.ref, 'refs/heads/u/jsickcodes/') + # as J.Sick Codes branches. + # if: > + # (startsWith(github.ref, 'refs/tags/') || startsWith(github.ref, 'refs/heads/tickets/')) || startsWith(github.ref, 'refs/heads/u/jsickcodes/') steps: - uses: actions/checkout@v2 + with: + fetch-depth: 0 - name: Define the Docker tag id: vars - run: echo ::set-output name=tag::$(bin/docker-tag.sh "$GITHUB_REF") + run: echo ::set-output name=tag::$(bin/docker-tag.sh) - name: Print the tag id: print @@ -123,33 +141,26 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v1 - - name: Cache Docker layers - uses: actions/cache@v2 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: - ${{ runner.os }}-buildx- - - name: Log in to Docker Hub uses: docker/login-action@v1 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_TOKEN }} + - name: Log in to GitHub Container Registry + uses: docker/login-action@v1 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push uses: docker/build-push-action@v2 with: context: . push: true - tags: lsstsqre/ltdkeeper:${{ steps.vars.outputs.tag }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache-new - - # Temp fix - # https://github.com/docker/build-push-action/issues/252 - # https://github.com/moby/buildkit/issues/1896 - - name: Move cache - run: | - rm -rf /tmp/.buildx-cache - mv /tmp/.buildx-cache-new /tmp/.buildx-cache + tags: | + lsstsqre/ltdkeeper:${{ steps.vars.outputs.tag }} + ghcr.io/lsst-sqre/ltd-keeper:${{ steps.vars.outputs.tag }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/Dockerfile b/Dockerfile index 279a0ab0..7cab2551 100644 --- a/Dockerfile +++ b/Dockerfile @@ -54,11 +54,14 @@ WORKDIR /home/appuser ENV PATH="/opt/venv/bin:$PATH" COPY --from=install-image /opt/venv /opt/venv -COPY uwsgi.ini bin migrations ./ +COPY bin uwsgi.ini ./ +ADD migrations ./migrations # Switch to non-root user USER appuser EXPOSE 3031 -CMD ["uwsgi", "uwsgi.ini"] +ENV FLASK_APP="keeper" + +CMD ["./start-api.bash"] diff --git a/bin/docker-tag.sh b/bin/docker-tag.sh index dbf61bb1..080a9c9b 100755 --- a/bin/docker-tag.sh +++ b/bin/docker-tag.sh @@ -1,13 +1,14 @@ #!/bin/bash -# Determine the tag for Docker images. Takes the Git ref as its only -# argument. +# Determine the tag for Docker images based on GitHub Actions environment +# variables. set -eo pipefail -if [ -z "$1" ]; then - echo 'Usage: scripts/docker-tag.sh $GITHUB_REF' >&2 - exit 1 +if [ -n "$GITHUB_HEAD_REF" ]; then + # For pull requests + echo ${GITHUB_HEAD_REF} | sed -E 's,/,-,g' +else + # For push events + echo ${GITHUB_REF} | sed -E 's,refs/(heads|tags)/,,' | sed -E 's,/,-,g' fi - -echo "$1" | sed -E 's,refs/(heads|tags)/,,' | sed -E 's,/,-,g' diff --git a/bin/install-base-packages.sh b/bin/install-base-packages.sh index fec16328..9713eb1e 100755 --- a/bin/install-base-packages.sh +++ b/bin/install-base-packages.sh @@ -29,7 +29,7 @@ apt-get -y upgrade # Install system packages # - build-essentiall needed for uwsgi # - git needed for setuptools_scm -apt-get -y install --no-install-recommends git build-essential +apt-get -y install --no-install-recommends git build-essential redis-server dnsutils wget # Delete cached files we don't need anymore: apt-get clean diff --git a/bin/start-api.bash b/bin/start-api.bash new file mode 100755 index 00000000..dc3b10a5 --- /dev/null +++ b/bin/start-api.bash @@ -0,0 +1,11 @@ +#!/bin/bash + +set -eu + +echo $PATH +pwd +ls migrations + +flask createdb migrations/alembic.ini +flask init +uwsgi uwsgi.ini diff --git a/keeper/cli.py b/keeper/cli.py index 5650e5cd..fe1e6d10 100644 --- a/keeper/cli.py +++ b/keeper/cli.py @@ -6,7 +6,6 @@ from __future__ import annotations -import os from typing import TYPE_CHECKING import alembic @@ -39,15 +38,16 @@ def add_app_commands(app: Flask) -> None: @click.command("createdb") +@click.argument("alembicconf") @with_appcontext -def createdb_command() -> None: +def createdb_command(alembicconf: str) -> None: """Deploy the current schema in a new database. This database is 'stamped' as having the current alembic schema version. Normally, in a new installation, run:: - flask createdb + flask createdb migrations/alembic.ini flask init This creates the tables and an initial user. @@ -57,10 +57,7 @@ def createdb_command() -> None: db.create_all() # stamp tables with latest schema version - config_path = os.path.abspath( - os.path.join(os.path.dirname(__file__), "..", "migrations/alembic.ini") - ) - alembic_cfg = alembic.config.Config(config_path) + alembic_cfg = alembic.config.Config(alembicconf) alembic.command.stamp(alembic_cfg, "head") diff --git a/manifests/base/keeper-cm.yaml b/manifests/base/keeper-cm.yaml new file mode 100644 index 00000000..9ed7bdee --- /dev/null +++ b/manifests/base/keeper-cm.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: ltd-keeper +data: + LTD_KEEPER_PROFILE: 'production' + LTD_KEEPER_URL_SCHEME: 'https' + LTD_KEEPER_BOOTSTRAP_USER: 'admin' + LTD_KEEPER_DB_URL: '' + REDIS_URL: '' + LTD_KEEPER_ENABLE_V1: '1' + LTD_KEEPER_ENABLE_V2: '1' + LTD_KEEPER_PROXY_FIX: '0' + LTD_KEEPER_X_FOR: '1' + LTD_KEEPER_X_PROTO: '1' + LTD_KEEPER_X_HOST: '1' + LTD_KEEPER_X_PORT: '0' + LTD_KEEPER_X_PREFIX: '0' + LTD_KEEPER_ENABLE_TASKS: '0' diff --git a/manifests/base/keeper-deployment.yaml b/manifests/base/keeper-deployment.yaml new file mode 100644 index 00000000..29ab6603 --- /dev/null +++ b/manifests/base/keeper-deployment.yaml @@ -0,0 +1,51 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ltd-keeper-api +spec: + replicas: 1 + selector: + matchLabels: + name: ltd-keeper-api + template: + metadata: + labels: + name: ltd-keeper-api + spec: + containers: + - name: app + imagePullPolicy: 'Always' + image: 'ghcr.io/lsst-sqre/ltd-keeper:latest' + ports: + - containerPort: 3031 + name: api + envFrom: + - configMapRef: + name: ltd-keeper + +--- +# Deployment of celery workers for keeper +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ltd-keeper-worker +spec: + replicas: 1 + selector: + matchLabels: + name: ltd-keeper-worker + template: + metadata: + labels: + name: ltd-keeper-worker + spec: + containers: + - name: app + imagePullPolicy: 'Always' + image: 'ghcr.io/lsst-sqre/ltd-keeper:latest' + command: ['/bin/bash'] + args: ['-c', '/home/appuser/run-celery-worker.bash'] + envFrom: + - configMapRef: + name: ltd-keeper diff --git a/manifests/base/keeper-service.yaml b/manifests/base/keeper-service.yaml new file mode 100644 index 00000000..dfc16845 --- /dev/null +++ b/manifests/base/keeper-service.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Service +metadata: + name: ltd-keeper + labels: + name: ltd-keeper +spec: + ports: + - name: ltd-keeper-http + protocol: TCP + port: 8080 + targetPort: api + selector: + name: ltd-keeper-api diff --git a/manifests/kustomization.yaml b/manifests/base/kustomization.yaml similarity index 61% rename from manifests/kustomization.yaml rename to manifests/base/kustomization.yaml index 66088806..ae73996d 100644 --- a/manifests/kustomization.yaml +++ b/manifests/base/kustomization.yaml @@ -3,11 +3,9 @@ kind: Kustomization resources: - keeper-cm.yaml - - redis-deployment.yaml - - redis-service.yaml - keeper-deployment.yaml - keeper-service.yaml images: - - name: lsstsqre/ltd-keeper - newTag: 1.20.3 + - name: 'lsstsqre/ltd-keeper:latest' + newTag: u-jsickcodes-deploy-2-0 diff --git a/manifests/keeper-cm.yaml b/manifests/keeper-cm.yaml deleted file mode 100644 index 533e4945..00000000 --- a/manifests/keeper-cm.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: keeper -data: - profile: "production" - # .. - cloud-sql-instance: "plasma-geode-127520:us-central1:ltd-sql-1" - url-scheme: "https" - # Cluster DNS url to LTD Dasher - dasher-url: "http://dasher:3031" - log-level: "INFO" diff --git a/manifests/keeper-deployment.yaml b/manifests/keeper-deployment.yaml deleted file mode 100644 index fb0c5542..00000000 --- a/manifests/keeper-deployment.yaml +++ /dev/null @@ -1,229 +0,0 @@ ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: keeper-api -spec: - replicas: 2 - selector: - matchLabels: - name: keeper-api - template: - metadata: - labels: - name: keeper-api - spec: - containers: - - - name: cloudsql-proxy - image: gcr.io/cloudsql-docker/gce-proxy:1.16 - command: ["/cloud_sql_proxy", "-dir=/cloudsql", "-credential_file=/secret/file.json", "-instances=$(CLOUD_SQL_INSTANCE)"] - env: - - name: CLOUD_SQL_INSTANCE - valueFrom: - configMapKeyRef: - name: keeper - key: cloud-sql-instance - volumeMounts: - - name: cloudsql - mountPath: /cloudsql - - name: cloudsql-secret-volume - mountPath: /secret/ - - name: cloudsql-ssl-certs - mountPath: /etc/ssl/certs - - - name: app - imagePullPolicy: "Always" - image: "lsstsqre/ltd-keeper:latest" - ports: - - containerPort: 3031 - name: keeper-uwsgi - volumeMounts: - - name: cloudsql - mountPath: /cloudsql - env: - # References the keeper-redis service - - name: REDIS_URL - value: "redis://keeper-redis:6379" - # Environment variables from the keeper configmap - - name: LTD_KEEPER_PROFILE - valueFrom: - configMapKeyRef: - name: keeper - key: profile - - name: LTD_KEEPER_URL_SCHEME - valueFrom: - configMapKeyRef: - name: keeper - key: url-scheme - - name: LTD_DASHER_URL - valueFrom: - configMapKeyRef: - name: keeper - key: dasher-url - # Environment variables from the keeper secret - - name: LTD_KEEPER_SECRET_KEY - valueFrom: - secretKeyRef: - name: keeper - key: secret-key - - name: LTD_KEEPER_AWS_ID - valueFrom: - secretKeyRef: - name: keeper - key: aws-id - - name: LTD_KEEPER_AWS_SECRET - valueFrom: - secretKeyRef: - name: keeper - key: aws-secret - - name: LTD_KEEPER_FASTLY_ID - valueFrom: - secretKeyRef: - name: keeper - key: fastly-id - - name: LTD_KEEPER_FASTLY_KEY - valueFrom: - secretKeyRef: - name: keeper - key: fastly-key - - name: LTD_KEEPER_BOOTSTRAP_USER - valueFrom: - secretKeyRef: - name: keeper - key: default-user - - name: LTD_KEEPER_BOOTSTRAP_PASSWORD - valueFrom: - secretKeyRef: - name: keeper - key: default-password - - name: LTD_KEEPER_DB_URL - valueFrom: - secretKeyRef: - name: keeper - key: db-url - - volumes: - - name: cloudsql-secret-volume - secret: - secretName: cloudsql - - name: cloudsql-ssl-certs - hostPath: - path: /etc/ssl/certs - - name: cloudsql - emptyDir: {} - ---- -# Deployment of celery workers for keeper -apiVersion: apps/v1 -kind: Deployment -metadata: - name: keeper-worker-deployment -spec: - replicas: 2 - selector: - matchLabels: - name: keeper-worker - template: - metadata: - labels: - name: keeper-worker - spec: - containers: - - - name: cloudsql-proxy - image: gcr.io/cloudsql-docker/gce-proxy:1.16 - command: ["/cloud_sql_proxy", "-dir=/cloudsql", "-credential_file=/secret/file.json", "-instances=$(CLOUD_SQL_INSTANCE)"] - env: - - name: CLOUD_SQL_INSTANCE - valueFrom: - configMapKeyRef: - name: keeper - key: cloud-sql-instance - volumeMounts: - - name: cloudsql - mountPath: /cloudsql - - name: cloudsql-secret-volume - mountPath: /secret/ - - name: cloudsql-ssl-certs - mountPath: /etc/ssl/certs - - - name: app - imagePullPolicy: "Always" - image: "lsstsqre/ltd-keeper:latest" - command: ["/bin/bash"] - args: ["-c", "/home/appuser/run-celery-worker.bash"] - volumeMounts: - - name: cloudsql - mountPath: /cloudsql - env: - # References the keeper-redis service - - name: REDIS_URL - value: "redis://keeper-redis:6379" - # Environment variables from the keeper configmap - - name: LTD_KEEPER_PROFILE - valueFrom: - configMapKeyRef: - name: keeper - key: profile - - name: LTD_KEEPER_URL_SCHEME - valueFrom: - configMapKeyRef: - name: keeper - key: url-scheme - - name: LTD_DASHER_URL - valueFrom: - configMapKeyRef: - name: keeper - key: dasher-url - # Environment variables from the keeper secret - - name: LTD_KEEPER_SECRET_KEY - valueFrom: - secretKeyRef: - name: keeper - key: secret-key - - name: LTD_KEEPER_AWS_ID - valueFrom: - secretKeyRef: - name: keeper - key: aws-id - - name: LTD_KEEPER_AWS_SECRET - valueFrom: - secretKeyRef: - name: keeper - key: aws-secret - - name: LTD_KEEPER_FASTLY_ID - valueFrom: - secretKeyRef: - name: keeper - key: fastly-id - - name: LTD_KEEPER_FASTLY_KEY - valueFrom: - secretKeyRef: - name: keeper - key: fastly-key - - name: LTD_KEEPER_BOOTSTRAP_USER - valueFrom: - secretKeyRef: - name: keeper - key: default-user - - name: LTD_KEEPER_BOOTSTRAP_PASSWORD - valueFrom: - secretKeyRef: - name: keeper - key: default-password - - name: LTD_KEEPER_DB_URL - valueFrom: - secretKeyRef: - name: keeper - key: db-url - - volumes: - - name: cloudsql-secret-volume - secret: - secretName: cloudsql - - name: cloudsql-ssl-certs - hostPath: - path: /etc/ssl/certs - - name: cloudsql - emptyDir: {} diff --git a/manifests/keeper-service.yaml b/manifests/keeper-service.yaml deleted file mode 100644 index 73e3cc27..00000000 --- a/manifests/keeper-service.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: keeper - labels: - name: keeper -spec: - ports: - - name: keeper-http - protocol: TCP - port: 8080 - targetPort: keeper-uwsgi - selector: - name: keeper-api diff --git a/manifests/redis-deployment.yaml b/manifests/redis-deployment.yaml deleted file mode 100644 index 2cb692bc..00000000 --- a/manifests/redis-deployment.yaml +++ /dev/null @@ -1,28 +0,0 @@ -# Deployment of a redis DB. -# -# This deployment isn't high-availability, nor does it have persistent storage. -# Those features may be added in the future (particularly by replacing this -# deployment with a HA Redis Helm chart. - -apiVersion: apps/v1 -kind: Deployment -metadata: - name: keeper-redis -spec: - replicas: 1 # Don't scale this deployment up - selector: - matchLabels: - name: keeper-redis - template: - metadata: - labels: - name: "keeper-redis" - spec: - containers: - - - name: "redis" - imagePullPolicy: "Always" - image: "redis:4-alpine" - ports: - - containerPort: 6379 - name: "keeper-redis" diff --git a/manifests/redis-service.yaml b/manifests/redis-service.yaml deleted file mode 100644 index 882bc350..00000000 --- a/manifests/redis-service.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# This Services provides a cluster endpoint for the redis pod. - -apiVersion: v1 -kind: Service -metadata: - name: keeper-redis - labels: - name: keeper-redis -spec: - type: ClusterIP - ports: - - name: keeper-redis - protocol: TCP - port: 6379 - targetPort: keeper-redis - selector: - name: keeper-redis