Merge pull request #347 from dbmi-bgm/c4_519

C4-519 Application Dockerization

.dockerignore

@@ -0,0 +1,8 @@
+.docker-compose.yml
+Dockerfile
+node_modules/
+.cache
+.pytest_cache
+aws-ip-ranges.json
+.git
+docs

.github/workflows/main.yml

@@ -25,7 +25,7 @@ jobs:
     # Build matrix
     strategy:
       matrix:
-        test_type: ['UNIT', 'NPM']
+        test_type: ['UNIT', 'NPM', 'Docker']
 
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
@@ -42,6 +42,7 @@ jobs:
           check-latest: false
 
       - name: Install/Link Postgres
+        if: ${{ matrix.test_type == 'NPM' || matrix.test_type == 'UNIT' }}
         run: |
           sudo apt-get install curl ca-certificates gnupg
           curl https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
@@ -51,6 +52,7 @@ jobs:
           echo "/usr/lib/postgresql/11/bin" >> $GITHUB_PATH
           sudo ln -s /usr/lib/postgresql/11/bin/initdb /usr/local/bin/initdb
       - name: Install Deps
+        if: ${{ matrix.test_type == 'NPM' || matrix.test_type == 'UNIT' }}
         run: |
           node --version
           make build
@@ -117,3 +119,7 @@ jobs:
           # Until the next version of snovault, the following two are prudent. We can remove them soon. -kmp 9-Mar-2021
           poetry run wipe-test-indices $TRAVIS_JOB_ID search-cgap-testing-6-8-vo4mdkmkshvmyddc65ux7dtaou.us-east-1.es.amazonaws.com:443
           poetry run wipe-test-indices cgap-test-$TRAVIS_JOB_ID search-cgap-testing-6-8-vo4mdkmkshvmyddc65ux7dtaou.us-east-1.es.amazonaws.com:443
+
+      - name: Docker Build
+        if: ${{ matrix.test_type == 'Docker' }}
+        run: make build-docker-local

.gitignore

@@ -82,3 +82,8 @@ elasticsearch-*.deb
 
 # Used for some kinds of debugging in dcicutils, snovault, cgap & ff.
 DEBUGLOG-*
+
+# Elastic Beanstalk Files
+.elasticbeanstalk/*
+!.elasticbeanstalk/*.cfg.yml
+!.elasticbeanstalk/*.global.yml

Dockerfile

@@ -0,0 +1,140 @@
+# CGAP-Portal (Production) Dockerfile
+# Note that images are pinned via sha256 as opposed to tag
+# so that we don't pick up new images unintentionally
+
+# Debian Buster with Python 3.6.13
+# TODO: maybe swap in ubuntu 20.04 and install Python manually?
+FROM python@sha256:db248d2d0494973550d323dd6b82af7fc2f4c1e0365769a758abd7fac2aa70db
+
+MAINTAINER William Ronchetti "[email protected]"
+
+# Build Arguments
+ARG INI_BASE
+ENV INI_BASE=${INI_BASE:-"cgap_any_alpha.ini"}
+
+# Configure (global) Env
+ENV NGINX_USER=nginx
+ENV DEBIAN_FRONTEND=noninteractive
+ENV CRYPTOGRAPHY_DONT_BUILD_RUST=1
+ENV PYTHONFAULTHANDLER=1 \
+  PYTHONUNBUFFERED=1 \
+  PYTHONHASHSEED=random \
+  PIP_NO_CACHE_DIR=off \
+  PIP_DISABLE_PIP_VERSION_CHECK=on \
+  PIP_DEFAULT_TIMEOUT=100 \
+  POETRY_VERSION=1.1.4 \
+  NODE_VERSION=12.22.1
+
+# Install nginx, base system
+COPY deploy/docker/production/install_nginx.sh /
+RUN bash /install_nginx.sh && \
+    apt-get update && \
+    apt-get install -y curl vim emacs postgresql-client net-tools ca-certificates
+
+# Configure CGAP User (nginx)
+WORKDIR /home/nginx/.nvm
+
+# Install Node
+ENV NVM_DIR=/home/nginx/.nvm
+RUN apt install -y curl
+RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.38.0/install.sh | bash
+RUN . "$NVM_DIR/nvm.sh" && nvm install ${NODE_VERSION}
+RUN . "$NVM_DIR/nvm.sh" && nvm use v${NODE_VERSION}
+RUN . "$NVM_DIR/nvm.sh" && nvm alias default v${NODE_VERSION}
+ENV PATH="/home/nginx/.nvm/versions/node/v${NODE_VERSION}/bin/:${PATH}"
+RUN node --version
+RUN npm --version
+
+WORKDIR /home/nginx
+
+# Configure venv
+ENV VIRTUAL_ENV=/opt/venv
+RUN python -m venv /opt/venv
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+
+# Upgrade pip, install in layer
+RUN pip install --upgrade pip && \
+    pip install poetry==1.1.4
+
+# Adjust permissions
+RUN chown -R nginx:nginx /opt/venv && \
+    mkdir -p /home/nginx/cgap-portal
+
+WORKDIR /home/nginx/cgap-portal
+
+# Do the back-end dependency install
+COPY pyproject.toml .
+COPY poetry.lock .
+RUN poetry install --no-root
+
+# Do the front-end dependency install
+COPY package.json .
+COPY package-lock.json .
+RUN npm ci --no-fund --no-progress --no-optional --no-audit --python=/opt/venv/bin/python
+
+# Copy over the rest of the code
+COPY . .
+
+# Build remaining back-end
+RUN poetry install && \
+    python setup_eb.py develop && \
+    make fix-dist-info
+
+# Build front-end
+RUN npm run build && \
+    npm run build-scss
+
+# Misc
+RUN make aws-ip-ranges && \
+    cat /dev/urandom | head -c 256 | base64 > session-secret.b64
+
+# Copy config files in (down here for quick debugging)
+# Remove default configuration from Nginx
+RUN rm /etc/nginx/nginx.conf && \
+    rm /etc/nginx/conf.d/default.conf
+COPY deploy/docker/production/nginx.conf /etc/nginx/nginx.conf
+
+# nginx filesystem setup
+RUN chown -R nginx:nginx /var/cache/nginx && \
+    chown -R nginx:nginx /var/log/nginx && \
+    chown -R nginx:nginx /etc/nginx/conf.d && \
+    touch /var/run/nginx.pid && \
+    chown -R nginx:nginx /var/run/nginx.pid && \
+    rm -f /var/log/nginx/* && \
+    touch /var/log/nginx/access.log && \
+    chown -R nginx:nginx /var/log/nginx/access.log && \
+    touch /var/log/nginx/error.log && \
+    chown -R nginx:nginx /var/log/nginx/error.log
+
+# Pull all required files
+# Note that *.ini must match the env name in secrets manager!
+# Note that deploy/docker/production/entrypoint.sh resolves which entrypoint to run
+# based on env variable "application_type".
+# For now, this is mastertest. - Will 04/29/21
+COPY deploy/docker/local/docker_development.ini development.ini
+COPY deploy/docker/local/entrypoint.sh entrypoint_local.sh
+RUN chown nginx:nginx development.ini
+RUN chmod +x entrypoint_local.sh
+
+# Production setup
+RUN touch production.ini
+RUN chown nginx:nginx production.ini
+COPY deploy/docker/production/$INI_BASE deploy/ini_files/.
+COPY deploy/docker/production/entrypoint.sh .
+COPY deploy/docker/production/entrypoint_portal.sh .
+COPY deploy/docker/production/entrypoint_deployment.sh .
+COPY deploy/docker/production/entrypoint_indexer.sh .
+COPY deploy/docker/production/entrypoint_ingester.sh .
+COPY deploy/docker/production/assume_identity.py .
+RUN chmod +x entrypoint.sh
+RUN chmod +x entrypoint_deployment.sh
+RUN chmod +x entrypoint_deployment.sh
+RUN chmod +x entrypoint_indexer.sh
+RUN chmod +x entrypoint_ingester.sh
+RUN chmod +x assume_identity.py
+EXPOSE 8000
+
+# Container does not run as root
+USER nginx
+
+ENTRYPOINT ["/home/nginx/cgap-portal/entrypoint.sh"]

Makefile

@@ -172,6 +172,49 @@ remote-test-unit:  # Note this does the 'indexing' tests
 update:  # updates dependencies
 	poetry update
 
+build-docker-local:
+	docker-compose build
+
+build-docker-local-clean:
+	docker-compose build --no-cache BUILD_PATH=deploy/docker/local
+
+deploy-docker-local:
+	docker-compose up -V
+
+deploy-docker-local-daemon:
+	docker-compose up -d -V
+
+ENV_NAME ?= cgap-mastertest
+AWS_ACCOUNT ?= 645819926742
+
+ecr-login:
+	@echo "Making ecr-login AWS_ACCOUNT=${AWS_ACCOUNT} ..."
+	aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin ${AWS_ACCOUNT}.dkr.ecr.us-east-1.amazonaws.com
+
+rebuild-docker-production:
+	@echo "Remaking build-docker-production AWS_ACCOUNT=${AWS_ACCOUNT} ENV_NAME=${ENV_NAME} ..."
+	docker build -t ${ENV_NAME}:latest . --no-cache
+	make tag-and-push-docker-production
+
+build-docker-test:
+	# This will do the equivalent of
+	#    make ecr-login AWS_ACCOUNT=<selected-test-account>
+	#    make build-docker-production AWS_ACCOUNT=<selected-test-account> ENV_NAME=<selected-env>
+	# but it has to do the login inside the script, we can't do it separately here
+	# because it has to infer the correct AWS_ACCOUNT and ENV_NAME by nosing into
+	# ~/.aws_test/test_creds.sh looking for ACCOUNT_NUMBER (note: not AWS_ACCOUNT) and ENV_NAME.
+	scripts/build-docker-test --login  # The login must be done inside the script, after inferring account number
+
+build-docker-production:
+	@echo "Making build-docker-production AWS_ACCOUNT=${AWS_ACCOUNT} ENV_NAME=${ENV_NAME} ..."
+	docker build -t ${ENV_NAME}:latest .
+	make tag-and-push-docker-production ENV_NAME=${ENV_NAME} AWS_ACCOUNT=${AWS_ACCOUNT}
+
+tag-and-push-docker-production:
+	@echo "Making tag-and-push-docker-production AWS_ACCOUNT=${AWS_ACCOUNT} ENV_NAME=${ENV_NAME} ..."
+	docker tag ${ENV_NAME}:latest ${AWS_ACCOUNT}.dkr.ecr.us-east-1.amazonaws.com/${ENV_NAME}:latest
+	docker push ${AWS_ACCOUNT}.dkr.ecr.us-east-1.amazonaws.com/${ENV_NAME}:latest
+
 help:
 	@make info
 
@@ -199,3 +242,9 @@ info:
 	   $(info - Use 'make test' to run tests with normal options similar to what we use on GitHub Actions.)
 	   $(info - Use 'make test-any' to run tests without marker constraints (i.e., with no '-m' option).)
 	   $(info - Use 'make update' to update dependencies (and the lock file).)
+	   $(info - Use 'make build-docker-local' to build the local Docker image.)
+	   $(info - Use 'make build-docker-local-clean' to build the local Docker image with no cache.)
+	   $(info - Use 'make deploy-docker-local' start up the cluster - pserve output will follow if successful.)
+	   $(info - Use 'make deploy-docker-local-daemon' will start the cluster in daemon mode.)
+	   $(info - Use 'make ecr-login' to login to ECR with the currently sourced AWS creds.)
+	   $(info - Use 'make build-docker-production' to build/tag/push a production image.)

README.rst

@@ -2,19 +2,7 @@
  CGAP PORTAL (HMS-BGM)
 ========================
 
-.. image:: https://travis-ci.org/dbmi-bgm/cgap-portal.svg?branch=master
-   :target: https://travis-ci.org/dbmi-bgm/cgap-portal
-
-|Coverage|_
-
-.. |Coverage| image:: https://coveralls.io/repos/github/4dn-dcic/fourfront/badge.svg?branch=master
-.. _Coverage: https://coveralls.io/github/4dn-dcic/fourfront?branch=master
-
-|Quality|_
-
-.. |Quality| image:: https://api.codacy.com/project/badge/Grade/f5fc54006b4740b5800e83eb2aeeeb43
-.. _Quality: https://www.codacy.com/app/4dn/fourfront?utm_source=github.com&utm_medium=referral&utm_content=4dn-dcic/fourfront&utm_campaign=Badge_Grade
-
+.. image:: https://github.com/dbmi-bgm/cgap-portal/actions/workflows/main.yml/badge.svg
 
 .. image:: https://readthedocs.org/projects/cgap-portal/badge/?version=latest
 
@@ -27,3 +15,47 @@ Welcome to CGAP! We are a team of scientists, clinicians, and developers who aim
 * `cgapwolf <http://fourfront-cgapwolf.9wzadzju3p.us-east-1.elasticbeanstalk.com/>`_ for workflow development
 
 Be warned that features are under active development and may not be stable! Visit the production deployment for the best experience. For installation and more information on getting started, see our `documentation page <https://cgap-portal.readthedocs.io/en/latest/index.html>`_.
+
+Note that at this time, CGAP is operating in hybrid model where some environments are deployed to AWS ElasticBeanstalk and others are deployed to AWS Elastic Container Service. The BS deployments are referred to as "legacy deployments" and the ECS deployments are referred to as "alpha deployments".
+
+For information on how to run CGAP with Docker, see `here. <./docs/source/docker-local.rst>`_
+
+For information on CGAP-Docker in production, see `here. <./docs/source/docker-production.rst>`_
+
+Navigating this Repository
+^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Important directories/files are outlined below.
+
+    * ``.github/workflows/`` contains Github Action Workflows
+    * ``.ebextensions/`` contains the Elastic Beanstalk provisioning scripts
+    * ``bin/`` contains the few remaining executables
+    * ``deploy/docker`` contains containerization related scripts/configuration
+    * ``docs/ contains`` documentation
+    * ``parts/`` contains WSGI entry points for the Beanstalk setup
+    * ``scripts/`` contains misc scripts
+    * ``src/encoded/`` where the code is
+    * ``.dockerignore`` specifies paths ignored by the Dockerfile
+    * ``Dockerfile`` contains the Docker build instructions for the cgap-portal - see ``docker-production.rst``
+    * ``Makefile`` contains macros for common build operations - see ``make info``
+    * ``docker-compose.yml`` builds the new local deployment - see ``docker-local.rst``
+    * ``package.json`` and ``package-lock.json`` specify the front-end dependencies
+    * ``pyproject.toml`` and ``poetry.lock`` specify the back-end dependencies
+    * ``setup_eb.py`` performs final installation setup
+
+Navigating src/encoded/
+^^^^^^^^^^^^^^^^^^^^^^^
+
+Top level files are modules that make up the core functionality of the back-end. Some modules differ greatly from or do
+not even exist in fourfront. Directories are outlined below.
+
+    * ``annotations/`` contains mapping table and ingestion related metadata
+    * ``commands/`` contains Python commands that can be run on the system from the command line
+    * ``docs/`` contains ReadTheDocs documentation
+    * ``ingestion/`` contains ingestion related code, such as mapping table intake and VCF processing
+    * ``schemas/`` contains the metadata schemas
+    * ``search/`` contains the search/filter_set APIs
+    * ``static/`` contains front-end code
+    * ``tests/`` contains back-end unit tests and insert data
+    * ``types/`` contains metadata type definitions
+    * ``upgrade/`` contains collection schema version upgraders - are not functioning as intended currently