Skip to content

Commit

Permalink
Update values files and githuba action env vars
Browse files Browse the repository at this point in the history
  • Loading branch information
Rub21 committed Oct 25, 2024
1 parent 69a247f commit c069e8e
Show file tree
Hide file tree
Showing 6 changed files with 338 additions and 140 deletions.
114 changes: 66 additions & 48 deletions .github/workflows/chartpress.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ on:
branches:
- 'main'
- 'staging'
- 'development'
jobs:
build:
runs-on: ubuntu-20.04
Expand All @@ -29,14 +30,57 @@ jobs:
run: chartpress --push
env:
GITHUB_TOKEN: ${{ secrets.GHCR_GITHUB_TOKEN }}
################ Development secrets ################
- name: Staging - substitute secrets
if: github.ref == 'refs/heads/staging'
if: github.ref == 'refs/heads/development'
uses: bluwy/substitute-string-action@v1
with:
_input-file: 'values.development.template.yaml'
_format-key: '{{key}}'
_output-file: 'values.development.yaml'
AWS_SSL_ARN: ${{ secrets.AWS_SSL_ARN }}
## web
MAILER_ADDRESS: ${{ secrets.MAILER_ADDRESS }}
MAILER_DOMAIN: ${{ secrets.MAILER_DOMAIN }}
MAILER_PASSWORD: ${{ secrets.STAGING_MAILER_PASSWORD }}
MAILER_USERNAME: ${{ secrets.STAGING_MAILER_USERNAME }}
DEVELOPMENT_DB: ${{ secrets.STAGING_DB }}
DEVELOPMENT_DB_EBS: ${{ secrets.STAGING_DB_EBS }}
DEVELOPMENT_DB_PASSWORD: ${{ secrets.STAGING_DB_PASSWORD }}
DEVELOPMENT_DB_USER: ${{ secrets.STAGING_DB_USER }}
DEVELOPMENT_DOMAIN_NAME: staging.openhistoricalmap.org
DEVELOPMENT_ID_KEY: ${{ secrets.STAGING_ID_KEY }}
DEVELOPMENT_ID_APPLICATION: ${{ secrets.STAGING_ID_APPLICATION }}
DEVELOPMENT_OAUTH_CLIENT_ID: ${{ secrets.STAGING_OAUTH_CLIENT_ID }}
DEVELOPMENT_OAUTH_KEY: ${{ secrets.STAGING_OAUTH_KEY }}
DEVELOPMENT_S3_BUCKET: osmseed-dev
## tiler
DEVELOPMENT_TILER_DB_PASSWORD: ${{ secrets.STAGING_TILER_DB_PASSWORD }}
DEVELOPMENT_TILER_CACHE_AWS_ACCESS_KEY_ID: ${{ secrets.STAGING_TILER_CACHE_AWS_ACCESS_KEY_ID }}
DEVELOPMENT_TILER_CACHE_AWS_SECRET_ACCESS_KEY: ${{ secrets.STAGING_TILER_CACHE_AWS_SECRET_ACCESS_KEY }}
## tm
DEVELOPMENT_TM_DB_PASSWORD: ${{ secrets.STAGING_TM_DB_PASSWORD }}
DEVELOPMENT_TM_API_SECRET: ${{ secrets.STAGING_TM_API_SECRET }}
DEVELOPMENT_TM_CLIENT_ID: ${{secrets.STAGING_TM_CLIENT_ID}}
DEVELOPMENT_TM_CLIENT_SECRET: ${{secrets.STAGING_TM_CLIENT_SECRET}}
## nominatim
DEVELOPMENT_NOMINATIM_PG_PASSWORD: ${{ secrets.STAGING_NOMINATIM_PG_PASSWORD }}
## osmcha
DEVELOPMENT_OSMCHA_PG_PASSWORD: ${{ secrets.STAGING_OSMCHA_PG_PASSWORD }}
DEVELOPMENT_OSMCHA_API_CONSUMER_KEY: ${{ secrets.STAGING_OSMCHA_API_CONSUMER_KEY }}
DEVELOPMENT_OSMCHA_API_CONSUMER_SECRET: ${{ secrets.STAGING_OSMCHA_API_CONSUMER_SECRET }}
DEVELOPMENT_OSMCHA_DJANGO_SECRET_KEY: ${{ secrets.STAGING_OSMCHA_DJANGO_SECRET_KEY }}
DEVELOPMENT_OSMCHA_REACT_APP_MAPBOX_ACCESS_TOKEN: ${{ secrets.STAGING_OSMCHA_REACT_APP_MAPBOX_ACCESS_TOKEN }}
################ Staging secrets ################
- name: Staging - substitute secrets
if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/development'
uses: bluwy/substitute-string-action@v1
with:
_input-file: 'values.staging.template.yaml'
_format-key: '{{key}}'
_output-file: 'values.staging.yaml'
AWS_SSL_ARN: ${{ secrets.AWS_SSL_ARN }}
## web
MAILER_ADDRESS: ${{ secrets.MAILER_ADDRESS }}
MAILER_DOMAIN: ${{ secrets.MAILER_DOMAIN }}
MAILER_PASSWORD: ${{ secrets.STAGING_MAILER_PASSWORD }}
Expand All @@ -51,41 +95,26 @@ jobs:
STAGING_OAUTH_CLIENT_ID: ${{ secrets.STAGING_OAUTH_CLIENT_ID }}
STAGING_OAUTH_KEY: ${{ secrets.STAGING_OAUTH_KEY }}
STAGING_S3_BUCKET: ${{ secrets.STAGING_S3_BUCKET }}
# STAGING_TILER_DB: ${{ secrets.STAGING_TILER_DB }}
# STAGING_TILER_DB_EBS: ${{ secrets.STAGING_TILER_DB_EBS }}
# STAGING_TILER_DB_HOST: ${{ secrets.STAGING_TILER_DB_HOST }}
## tiler
STAGING_TILER_DB_PASSWORD: ${{ secrets.STAGING_TILER_DB_PASSWORD }}
# STAGING_TILER_DB_USER: ${{ secrets.STAGING_TILER_DB_USER }}
# STAGING_TILER_IMPOSM_EBS: ${{ secrets.STAGING_TILER_IMPOSM_EBS }}
# STAGING_TILER_IMPOSM_REPLICATION_URL: ${{ secrets.STAGING_TILER_IMPOSM_REPLICATION_URL }}
# STAGING_TILER_SERVER_EBS: ${{ secrets.STAGING_TILER_SERVER_EBS }}
STAGING_TILER_CACHE_AWS_ACCESS_KEY_ID: ${{ secrets.STAGING_TILER_CACHE_AWS_ACCESS_KEY_ID }}
STAGING_TILER_CACHE_AWS_SECRET_ACCESS_KEY: ${{ secrets.STAGING_TILER_CACHE_AWS_SECRET_ACCESS_KEY }}
## tm
STAGING_TM_API_CONSUMER_KEY: ${{ secrets.STAGING_TM_API_CONSUMER_KEY }}
STAGING_TM_API_CONSUMER_SECRET: ${{ secrets.STAGING_TM_API_CONSUMER_SECRET }}
# STAGING_TM_API_DB: ${{ secrets.STAGING_TM_API_DB }}
# STAGING_TM_API_DB_HOST: ${{ secrets.STAGING_TM_API_DB_HOST }}
# STAGING_TM_API_DB_PASSWORD: ${{ secrets.STAGING_TM_API_DB_PASSWORD }}
# STAGING_TM_API_DB_USER: ${{ secrets.STAGING_TM_API_DB_USER }}
STAGING_TM_DB_PASSWORD: ${{ secrets.STAGING_TM_DB_PASSWORD }}
STAGING_TM_API_SECRET: ${{ secrets.STAGING_TM_API_SECRET }}
STAGING_TM_CLIENT_ID: ${{secrets.STAGING_TM_CLIENT_ID}}
STAGING_TM_CLIENT_SECRET: ${{secrets.STAGING_TM_CLIENT_SECRET}}
# STAGING_NOMINATIM_DB_EBS: ${{ secrets.STAGING_NOMINATIM_DB_EBS }}
# STAGING_NOMINATIM_PG_PORT: ${{ secrets.STAGING_NOMINATIM_PG_PORT }}
# STAGING_NOMINATIM_PG_USER: ${{ secrets.STAGING_NOMINATIM_PG_USER }}
## nominatim
STAGING_NOMINATIM_PG_PASSWORD: ${{ secrets.STAGING_NOMINATIM_PG_PASSWORD }}
# STAGING_NOMINATIM_PG_DATABASE: ${{ secrets.STAGING_NOMINATIM_PG_DATABASE }}
# STAGING_OVERPASS_API_DB_EBS: ${{ secrets.STAGING_OVERPASS_API_DB_EBS }}
STAGING_NEW_RELIC_LICENSE_KEY: ${{ secrets.STAGING_NEW_RELIC_LICENSE_KEY }}
STAGING_NEW_RELIC_APP_NAME: ${{ secrets.STAGING_NEW_RELIC_APP_NAME }}
STAGING_OSMCHA_PG_USER: ${{ secrets.STAGING_OSMCHA_PG_USER }}
## osmcha
STAGING_OSMCHA_PG_PASSWORD: ${{ secrets.STAGING_OSMCHA_PG_PASSWORD }}
STAGING_OSMCHA_PG_DATABASE: ${{ secrets.STAGING_OSMCHA_PG_DATABASE }}
STAGING_OSMCHA_API_CONSUMER_KEY: ${{ secrets.STAGING_OSMCHA_API_CONSUMER_KEY }}
STAGING_OSMCHA_API_CONSUMER_SECRET: ${{ secrets.STAGING_OSMCHA_API_CONSUMER_SECRET }}
STAGING_OSMCHA_DJANGO_SECRET_KEY: ${{ secrets.STAGING_OSMCHA_DJANGO_SECRET_KEY }}
STAGING_OSMCHA_REACT_APP_MAPBOX_ACCESS_TOKEN: ${{ secrets.STAGING_OSMCHA_REACT_APP_MAPBOX_ACCESS_TOKEN }}
STAGING_TM_DB_PASSWORD: ${{ secrets.STAGING_TM_DB_PASSWORD }}
################ Production secrets ################
- name: Production - substitute secrets
if: github.ref == 'refs/heads/main'
uses: bluwy/substitute-string-action@v1
Expand All @@ -94,6 +123,7 @@ jobs:
_format-key: '{{key}}'
_output-file: 'values.production.yaml'
AWS_SSL_ARN: ${{ secrets.AWS_SSL_ARN }}
## web
MAILER_ADDRESS: ${{ secrets.MAILER_ADDRESS }}
MAILER_DOMAIN: ${{ secrets.MAILER_DOMAIN }}
MAILER_PASSWORD: ${{ secrets.MAILER_PASSWORD }}
Expand All @@ -109,63 +139,51 @@ jobs:
PRODUCTION_OAUTH_KEY: ${{ secrets.PRODUCTION_OAUTH_KEY }}
PRODUCTION_S3_BUCKET: ${{ secrets.PRODUCTION_S3_BUCKET }}
PRODUCTION_DB_BACKUP_S3_BUCKET: ${{ secrets.PRODUCTION_DB_BACKUP_S3_BUCKET }}
# PRODUCTION_TILER_DB: ${{ secrets.PRODUCTION_TILER_DB }}
# PRODUCTION_TILER_DB_EBS: ${{ secrets.PRODUCTION_TILER_DB_EBS }}
# PRODUCTION_TILER_DB_HOST: ${{ secrets.PRODUCTION_TILER_DB_HOST }}
## tiler
PRODUCTION_TILER_DB_PASSWORD: ${{ secrets.PRODUCTION_TILER_DB_PASSWORD }}
# PRODUCTION_TILER_DB_USER: ${{ secrets.PRODUCTION_TILER_DB_USER }}
# PRODUCTION_TILER_IMPOSM_EBS: ${{ secrets.PRODUCTION_TILER_IMPOSM_EBS }}
# PRODUCTION_TILER_IMPOSM_REPLICATION_URL: ${{ secrets.PRODUCTION_TILER_IMPOSM_REPLICATION_URL }}
# PRODUCTION_TILER_SERVER_EBS: ${{ secrets.PRODUCTION_TILER_SERVER_EBS }}
PRODUCTION_TILER_CACHE_AWS_ACCESS_KEY_ID: ${{ secrets.PRODUCTION_TILER_CACHE_AWS_ACCESS_KEY_ID }}
## tm
PRODUCTION_TILER_CACHE_AWS_SECRET_ACCESS_KEY: ${{ secrets.PRODUCTION_TILER_CACHE_AWS_SECRET_ACCESS_KEY }}
PRODUCTION_TM_API_CONSUMER_KEY: ${{ secrets.PRODUCTION_TM_API_CONSUMER_KEY }}
PRODUCTION_TM_API_CONSUMER_SECRET: ${{ secrets.PRODUCTION_TM_API_CONSUMER_SECRET }}
# PRODUCTION_TM_API_DB: ${{ secrets.PRODUCTION_TM_API_DB }}
# PRODUCTION_TM_API_DB_HOST: ${{ secrets.PRODUCTION_TM_API_DB_HOST }}
# PRODUCTION_TM_API_DB_PASSWORD: ${{ secrets.PRODUCTION_TM_API_DB_PASSWORD }}
# PRODUCTION_TM_API_DB_USER: ${{ secrets.PRODUCTION_TM_API_DB_USER }}
PRODUCTION_TM_DB_PASSWORD: ${{ secrets.PRODUCTION_TM_DB_PASSWORD }}
PRODUCTION_TM_API_SECRET: ${{ secrets.PRODUCTION_TM_API_SECRET }}
PRODUCTION_TM_CLIENT_ID: ${{secrets.PRODUCTION_TM_CLIENT_ID}}
PRODUCTION_TM_CLIENT_SECRET: ${{secrets.PRODUCTION_TM_CLIENT_SECRET}}
PRODUCTION_NOMINATIM_DB_EBS: ${{ secrets.PRODUCTION_NOMINATIM_DB_EBS }}
PRODUCTION_NOMINATIM_PG_PORT: ${{ secrets.PRODUCTION_NOMINATIM_PG_PORT }}
PRODUCTION_NOMINATIM_PG_USER: ${{ secrets.PRODUCTION_NOMINATIM_PG_USER }}
## nominatim
PRODUCTION_NOMINATIM_PG_PASSWORD: ${{ secrets.PRODUCTION_NOMINATIM_PG_PASSWORD }}
PRODUCTION_NOMINATIM_PG_DATABASE: ${{ secrets.PRODUCTION_NOMINATIM_PG_DATABASE }}
PRODUCTION_OVERPASS_API_DB_EBS: ${{ secrets.PRODUCTION_OVERPASS_API_DB_EBS }}
PRODUCTION_NEW_RELIC_LICENSE_KEY: ${{ secrets.PRODUCTION_NEW_RELIC_LICENSE_KEY }}
PRODUCTION_NEW_RELIC_APP_NAME: ${{ secrets.PRODUCTION_NEW_RELIC_APP_NAME }}
PRODUCTION_OSMCHA_PG_USER: ${{ secrets.PRODUCTION_OSMCHA_PG_USER }}
## osmcha
PRODUCTION_OSMCHA_PG_PASSWORD: ${{ secrets.PRODUCTION_OSMCHA_PG_PASSWORD }}
PRODUCTION_OSMCHA_PG_DATABASE: ${{ secrets.PRODUCTION_OSMCHA_PG_DATABASE }}
PRODUCTION_OSMCHA_API_CONSUMER_KEY: ${{ secrets.PRODUCTION_OSMCHA_API_CONSUMER_KEY }}
PRODUCTION_OSMCHA_API_CONSUMER_SECRET: ${{ secrets.PRODUCTION_OSMCHA_API_CONSUMER_SECRET }}
PRODUCTION_OSMCHA_DJANGO_SECRET_KEY: ${{ secrets.PRODUCTION_OSMCHA_DJANGO_SECRET_KEY }}
PRODUCTION_OSMCHA_REACT_APP_MAPBOX_ACCESS_TOKEN: ${{ secrets.PRODUCTION_OSMCHA_REACT_APP_MAPBOX_ACCESS_TOKEN }}
PRODUCTION_TM_DB_PASSWORD: ${{ secrets.PRODUCTION_TM_DB_PASSWORD }}

- name: AWS Credentials
if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main'
if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/development'
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: us-east-1
- name: Setup Kubectl and Helm Dependencies
if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main'
if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/development'
run: "sudo pip install awscli --ignore-installed six\nsudo curl -L -o /usr/bin/kubectl https://amazon-eks.s3.us-west-2.amazonaws.com/1.17.7/2020-07-08/bin/linux/amd64/kubectl\nsudo chmod +x /usr/bin/kubectl\nsudo curl -o /usr/bin/aws-iam-authenticator https://amazon-eks.s3.us-west-2.amazonaws.com/1.17.7/2020-07-08/bin/linux/amd64/aws-iam-authenticator\nsudo chmod +x /usr/bin/aws-iam-authenticator\nwget https://get.helm.sh/helm-v3.5.0-linux-amd64.tar.gz -O helm.tar.gz\ntar -xvzf helm.tar.gz\nsudo mv linux-amd64/helm /usr/local/bin/ \nsudo chmod +x /usr/local/bin/helm\n #magic___^_^___line\n"
- name: Update kube-config staging
if: github.ref == 'refs/heads/staging'
- name: Update kube-config staging
if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/development'
run: aws eks --region us-east-1 update-kubeconfig --name osmseed-staging
- name: Update kube-config prod
if: github.ref == 'refs/heads/main'
run: aws eks --region us-east-1 update-kubeconfig --name osmseed-production-v2
- name: Install helm dependencies for
if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main'
if: github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/main' || github.ref == 'refs/heads/development'
run: cd ohm && helm dep up
- name: Staging - helm deploy
if: github.ref == 'refs/heads/staging'
run: helm upgrade --install staging --wait ohm/ -f values.staging.yaml -f ohm/values.yaml
# - name: development - helm deploy
# if: github.ref == 'refs/heads/development'
# run: helm upgrade --install development --wait ohm/ -f values.staging.yaml -f ohm/values.yaml
- name: Production - helm deploy
if: github.ref == 'refs/heads/main'
run: helm upgrade --install production --wait ohm/ -f values.production.yaml -f ohm/values.yaml
136 changes: 102 additions & 34 deletions images/tasking-manager-api/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,34 +1,102 @@
FROM python:3.7-slim

RUN apt update && \
apt install -y git

ENV workdir /usr/src/app

RUN git clone https://github.com/OpenHistoricalMap/tasking-manager.git $workdir
# Commits on 27 April, 2022
RUN cd $workdir && git checkout -f f0df07174f4014365220af09187b5f941f9770b0
WORKDIR $workdir

# Setup backend dependencies
RUN apt update && apt install -y \
gcc \
g++ \
make \
libffi-dev \
python3-dev \
libpq-dev \
proj-bin && \
apt clean && \
rm -rf /var/lib/apt/lists/*

RUN pip install --upgrade pip
RUN pip install -r requirements.txt
RUN pip install apscheduler==3.7.0
RUN pip install --upgrade markdown

## INITIALIZATION
EXPOSE 5000
CMD ["gunicorn", "-b", "0.0.0.0:5000", "--worker-class", "gevent", "--workers", "3", \
"--threads", "3", "--timeout", "179", "manage:application", "&"]

ARG DEBIAN_IMG_TAG=slim-bookworm
ARG PYTHON_IMG_TAG=3.10

FROM docker.io/python:${PYTHON_IMG_TAG}-${DEBIAN_IMG_TAG} as base
ARG APP_VERSION=0.1.0
ARG DOCKERFILE_VERSION=0.5.0
ARG ALPINE_IMG_TAG
ARG PYTHON_IMG_TAG
ARG [email protected]
LABEL org.hotosm.tasks.app-version="${APP_VERSION}" \
org.hotosm.tasks.debian-img-tag="${DEBIAN_IMG_TAG}" \
org.hotosm.tasks.python-img-tag="${PYTHON_IMG_TAG}" \
org.hotosm.tasks.dockerfile-version="${DOCKERFILE_VERSION}" \
org.hotosm.tasks.maintainer="${MAINTAINER}" \
org.hotosm.tasks.api-port="5000"
# Fix timezone (do not change - see issue #3638)
ENV TZ UTC
# Add non-root user, permissions, init log dir
RUN useradd --uid 9000 --create-home --home /home/appuser --shell /bin/false appuser

FROM base as extract-deps
RUN pip install --no-cache-dir --upgrade pip
WORKDIR /opt/python
COPY pyproject.toml pdm.lock README.md /opt/python/
RUN pip install --no-cache-dir pdm==2.7.4
RUN pdm export --prod --without-hashes > requirements.txt



FROM base as build
RUN pip install --no-cache-dir --upgrade pip
WORKDIR /opt/python
# Setup backend build-time dependencies
RUN apt-get update
RUN apt-get install --no-install-recommends -y build-essential
RUN apt-get install --no-install-recommends -y \
postgresql-server-dev-15 \
python3-dev \
libffi-dev \
libgeos-dev
# Setup backend Python dependencies
COPY --from=extract-deps \
/opt/python/requirements.txt /opt/python/
USER appuser:appuser
RUN pip install --user --no-warn-script-location \
--no-cache-dir -r /opt/python/requirements.txt



FROM base as runtime
ARG PYTHON_IMG_TAG
WORKDIR /usr/src/app
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
PYTHONFAULTHANDLER=1 \
PATH="/home/appuser/.local/bin:$PATH" \
PYTHON_LIB="/home/appuser/.local/lib/python$PYTHON_IMG_TAG/site-packages" \
SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt \
REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt
# Setup backend runtime dependencies
RUN apt-get update && \
apt-get install --no-install-recommends -y \
postgresql-client git libgeos3.11.1 proj-bin && \
apt-get clean && rm -rf /var/lib/apt/lists/*
COPY --from=build \
/home/appuser/.local \
/home/appuser/.local
USER appuser:appuser

# Clone the repository as root to avoid permission issues
USER root
RUN git clone https://github.com/OpenHistoricalMap/tasking-manager.git /usr/src/app && \
chown -R appuser:appuser /usr/src/app

# Switch back to non-root user
USER appuser:appuser

RUN cd /usr/src/app && git checkout -f 321a9c19e07576a3c3c2eff3df14ba48c7325837

FROM runtime as debug
RUN pip install --user --no-warn-script-location \
--no-cache-dir debugpy==1.6.7
EXPOSE 5678/tcp
CMD ["python", "-m", "debugpy", "--wait-for-client", "--listen", "0.0.0.0:5678", \
"-m", "gunicorn", "-c", "python:backend.gunicorn", "manage:application", \
"--reload", "--log-level", "error"]



FROM runtime as prod
USER root
RUN apt-get update && \
apt-get install -y curl && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*
# Pre-compile packages to .pyc (init speed gains)
RUN python -c "import compileall; compileall.compile_path(maxlevels=10, quiet=1)"
RUN python -m compileall .
EXPOSE 5000/tcp
USER appuser:appuser
CMD ["gunicorn", "-c", "python:backend.gunicorn", "manage:application", \
"--workers", "1", "--log-level", "error"]
2 changes: 1 addition & 1 deletion ohm/requirements.yaml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
dependencies:
- name: osm-seed
version: '0.1.0-n797.h36f9725'
version: '0.1.0-n798.h6499ceb'
repository: https://devseed.com/osm-seed-chart/
Loading

0 comments on commit c069e8e

Please sign in to comment.