add deployment setup

.github/workflows/deploy.yml

@@ -0,0 +1,67 @@
+name: Deploy to AWS EC2 with Terraform
+
+on:
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: 'Deployment environment'
+        required: true
+        default: staging
+        type: choice
+        options:
+          - staging
+  push:
+    tags:
+      - 'v*.*.*' # Matches version tags like v1.0.0
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          install: true
+
+      - name: Cache Docker layers
+        uses: actions/cache@v3
+        with:
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys: |
+            ${{ runner.os }}-buildx-
+
+      - name: Log in to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v2
+
+      - name: Build and push Docker image
+        run: |
+          DOCKER_BUILDKIT=1 docker build -t osm_web_api:${{ github.event.inputs.environment || 'production' }} -f ./docker_images/web_api/Dockerfile .
+          docker tag osm_web_api:${{ github.event.inputs.environment || 'production' }}:latest ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com/osm_web_api:${{ github.event.inputs.environment || 'production' }}
+          docker push ${{ secrets.AWS_ACCOUNT_ID }}.dkr.ecr.${{ secrets.AWS_REGION }}.amazonaws.com/osm_web_api:${{ github.event.inputs.environment || 'production' }}
+
+      - name: Setup Terraform
+        uses: hashicorp/setup-terraform@v2
+
+      - name: Terraform Init
+        run: terraform init -backend-config="path/to/${{ github.event.inputs.environment || 'production' }}/backend-config"
+
+      - name: Terraform Plan
+        run: terraform plan -var-file="terraform/${{ github.event.inputs.environment || 'production' }}.tfvars"
+
+      - name: Terraform Apply
+        if: success()
+        run: terraform apply -var-file="terraform/${{ github.event.inputs.environment || 'production' }}.tfvars" -auto-approve
+
+      - name: Notify Success
+        if: success()
+        run: echo "Deployment to ${{ github.event.inputs.environment || 'production' }} environment was successful."
+
+      - name: Notify Failure
+        if: failure()
+        run: echo "Deployment to ${{ github.event.inputs.environment || 'production' }} environment failed."

.gitignore

@@ -10,6 +10,7 @@ __pycache__/
 *.pyd
 *.env
 *.egg-info/
+*.tfstate
 dist/
 build/
 .tox/

web_api/.env.template

@@ -0,0 +1,11 @@
+# AWS_ACCESS_KEY_ID=
+# AWS_SECRET_ACCESS_KEY=
+# AWS_REGION="us-east-1"
+# AWS_ACCOUNT_ID="your-aws-account-id"
+# DOCKER_HUB_USERNAME="your-docker-hub-username"
+# DOCKER_HUB_ACCESS_TOKEN="your-docker-hub-access-token"
+# ENVIRONMENT=  # staging or production
+# DOCKER_IMAGE_TAG="$DOCKER_HUB_USERNAME/osm_web_api:$ENVIRONMENT"
+# S3_BUCKET="osm-storage"
+# MONGODB_URI="your-mongodb-uri"
+# DOMAIN="osm.nimh.nih.gov"

docker_images/web_api/Dockerfile → web_api/Dockerfile

@@ -9,7 +9,8 @@ RUN pip install --no-cache-dir --upgrade -r /app/app/requirements.txt
 RUN mkdir -p /opt/osm
 COPY pyproject.toml /opt/osm
 COPY osm /opt/osm/osm
-COPY .git /opt/osm/.git
-RUN pip install  /opt/osm
+ARG PSEUDO_VERSION=0.0.1 # strongly recommended to update based on git describe
+RUN SETUPTOOLS_SCM_PRETEND_VERSION_FOR_OSM=${PSEUDO_VERSION} pip install -e /opt/osm
+RUN --mount=source=.git,target=/opt/osm/.git,type=bind pip install -e /opt/osm
 
 COPY ./docker_images/web_api/main.py /app/app/main.py

web_api/README.md

@@ -0,0 +1,41 @@
+Guidance for deployment using Terraform and Docker Compose and GitHub Actions.
+
+### Files and Specifications
+
+1. **Terraform Configuration**
+   - **Purpose**: Provision EC2 instances and related infrastructure for the application and Traefik, with separate configurations for staging and production.
+   - **Resources**:
+     - EC2 instance (reserved instance) running Ubuntu.
+     - Security groups to allow necessary traffic (HTTPS and SSH on a non-standard port).
+     - Subnet configuration to place the instance in the correct network.
+     - DynamoDB table for Terraform state locking.
+
+2. **Docker Compose Configuration**
+   - **Purpose**: Manage the deployment of the application and Traefik on the EC2 instance.
+   - **Services**:
+     - **Traefik**: Acts as a TLS termination proxy, handling HTTPS traffic and forwarding it to the application.
+     - **Application**: The main application service running the Docker container built from the Dockerfile.
+
+3. **Terraform Backend Configuration**
+   - **Purpose**: Configure Terraform to use an S3 bucket for state storage and a DynamoDB table for state locking.
+   - **Resources**:
+     - S3 bucket configuration for storing Terraform state files.
+     - DynamoDB table for state locking.
+
+4. **Deployment Script (deploy.sh)**
+   - **Purpose**: Automate the deployment process for local development (staging environment).
+   - **Steps**:
+     - Set environment variables.
+     - Log in to Docker Hub.
+     - Build and push the Docker image.
+     - Initialize and apply Terraform configuration.
+     - Deploy Docker Compose configuration on the provisioned EC2 instance.
+
+5. **GitHub Actions Workflow (deploy.yml)**
+   - **Purpose**: Formal and automated deployment for staging and production environments.
+   - **Steps**:
+     - Checkout code.
+     - Log in to Docker Hub.
+     - Build and push the Docker image.
+     - Initialize and apply Terraform configuration.
+     - Deploy Docker Compose configuration on the provisioned EC2 instance.

docker_images/web_api/compose.yaml → web_api/compose.yaml

@@ -8,3 +8,12 @@ services:
       dockerfile: ./docker_images/web_api/Dockerfile
     ports:
       - 80:80
+  traefik:
+    image: traefik:v3.1
+    volumes:
+      - "/var/run/docker.sock:/var/run/docker.sock"
+      - "./traefik:/etc/traefik"
+      - "./letsencrypt:/letsencrypt"
+    ports:
+      - "80:80"
+      - "443:443"

web_api/deploy.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+# Set your variables here
+
+# Export AWS credentials
+export AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID
+export AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY
+
+# Log in to Docker Hub
+echo "Logging in to Docker Hub..."
+echo $DOCKER_HUB_ACCESS_TOKEN | docker login --username $DOCKER_HUB_USERNAME --password-stdin
+
+# Build and push Docker image
+echo "Building and pushing Docker image..."
+DOCKER_BUILDKIT=1 docker build -t $DOCKER_IMAGE_TAG -f ./web_api/docker_images/web_api/Dockerfile .
+docker push $DOCKER_IMAGE_TAG
+
+# Set up Terraform
+echo "Setting up Terraform..."
+terraform -install-autocomplete
+
+# Initialize Terraform
+echo "Initializing Terraform..."
+terraform init -backend-config="./web_api/terraform/backend-config.tf"
+
+# Plan Terraform changes
+echo "Planning Terraform changes..."
+terraform plan -var-file="./web_api/terraform/$ENVIRONMENT.tfvars"
+
+# Apply Terraform changes
+echo "Applying Terraform changes..."
+terraform apply -var-file="./web_api/terraform/$ENVIRONMENT.tfvars" -auto-approve
+
+# Get the instance ID and public DNS
+instance_id=$(terraform output -raw instance_id)
+public_dns=$(terraform output -raw public_dns)
+
+# Deploy Docker Compose on the instance
+echo "Deploying Docker Compose on the instance..."
+ssh -o StrictHostKeyChecking=no -i dsst2023.pem ubuntu@$public_dns -p 2222 << EOF
+  mkdir -p ~/app
+  cd ~/app
+  echo "$(<./web_api/docker-compose.yml)" > docker-compose.yml
+  export ENVIRONMENT=$ENVIRONMENT
+  export MONGODB_URI=$MONGODB_URI
+  docker-compose up -d
+EOF
+
+# Notify success
+if [ $? -eq 0 ]; then
+  echo "Deployment to $ENVIRONMENT environment was successful."
+else
+  echo "Deployment to $ENVIRONMENT environment failed."
+fi

web_api/terraform/backend-config.tf

@@ -0,0 +1,8 @@
+terraform {
+  backend "s3" {
+    bucket         = "osm-storage"
+    key            = "terraform/${ENVIRONMENT}/terraform.tfstate"
+    region         = "us-east-1"
+    dynamodb_table = "terraform-locks"
+  }
+}

web_api/terraform/main.tf

@@ -0,0 +1,123 @@
+provider "aws" {
+  region = "us-east-1"
+}
+
+variable "environment" {
+  description = "The environment to deploy to (staging or production)"
+  type        = string
+}
+
+variable "instance_type" {
+  default = "t3.micro"
+}
+
+variable "ssh_port" {
+  description = "Non-standard port for SSH"
+  default     = 2222
+}
+
+variable "s3_bucket" {
+  description = "S3 bucket for Terraform state"
+  default     = "osm-storage"
+}
+
+variable "dynamodb_table" {
+  description = "DynamoDB table for Terraform state locking"
+  default     = "terraform-locks"
+}
+
+variable "domain" {
+  description = "Domain for Traefik"
+  default     = "osm.nimh.nih.gov"
+}
+
+data "aws_ami" "ubuntu" {
+  most_recent = true
+  owners      = ["099720109477"]
+  filter {
+    name   = "name"
+    values = ["ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-*"]
+  }
+}
+
+resource "aws_vpc" "main" {
+  cidr_block = "10.0.0.0/16"
+
+  tags = {
+    Name = "${var.environment}-vpc"
+  }
+}
+
+resource "aws_subnet" "main" {
+  vpc_id            = aws_vpc.main.id
+  cidr_block        = "10.0.1.0/24"
+  availability_zone = "us-east-1a"
+
+  tags = {
+    Name = "${var.environment}-subnet"
+  }
+}
+
+resource "aws_security_group" "app" {
+  vpc_id = aws_vpc.main.id
+
+  ingress {
+    from_port   = 443
+    to_port     = 443
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  ingress {
+    from_port   = var.ssh_port
+    to_port     = var.ssh_port
+    protocol    = "tcp"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  egress {
+    from_port   = 0
+    to_port     = 0
+    protocol    = "-1"
+    cidr_blocks = ["0.0.0.0/0"]
+  }
+
+  tags = {
+    Name = "${var.environment}-security-group"
+  }
+}
+
+resource "aws_instance" "app" {
+  ami           = data.aws_ami.ubuntu.id
+  instance_type = var.instance_type
+  subnet_id     = aws_subnet.main.id
+  key_name      = "dsst2023"
+  security_groups = [aws_security_group.app.id]
+
+  tags = {
+    Name = "${var.environment}-instance"
+  }
+
+  user_data = <<-EOF
+              #!/bin/bash
+              apt-get update -y
+              apt-get install -y docker.io docker-compose
+              systemctl start docker
+              systemctl enable docker
+              EOF
+}
+
+resource "aws_dynamodb_table" "terraform_locks" {
+  name         = var.dynamodb_table
+  billing_mode = "PAY_PER_REQUEST"
+  hash_key     = "LockID"
+
+  attribute {
+    name = "LockID"
+    type = "S"
+  }
+
+  tags = {
+    Name = "${var.environment}-terraform-locks"
+  }
+}

web_api/terraform/production.tfvars

@@ -0,0 +1,3 @@
+environment = "production"
+subnet_ids = ["subnet-yyyyyyyy"]
+security_group_ids = ["sg-yyyyyyyy"]

web_api/terraform/staging.tfvars

@@ -0,0 +1,3 @@
+environment = "staging"
+subnet_ids = ["subnet-xxxxxxxx"]
+security_group_ids = ["sg-xxxxxxxx"]

web_api/terraform/variables.tf

@@ -0,0 +1,42 @@
+variable "aws_region" {
+  description = "AWS region"
+  default     = "us-east-1"
+}
+
+variable "environment" {
+  description = "Deployment environment (staging or production)"
+  type        = string
+}
+
+variable "instance_type" {
+  description = "EC2 instance type"
+  default     = "t3.micro"
+}
+
+variable "spot_instance_type" {
+  description = "EC2 spot instance type"
+  default     = "t3.micro"
+}
+
+variable "subnet_ids" {
+  description = "List of subnet IDs"
+  type        = list(string)
+}
+
+variable "security_group_ids" {
+  description = "List of security group IDs"
+  type        = list(string)
+}
+
+variable "mongodb_uri" {
+  description = "MongoDB URI for state locking"
+}
+
+variable "mongodb_db" {
+  description = "MongoDB Database for state locking"
+}
+
+variable "domain" {
+  description = "Domain for Traefik"
+  default     = ""
+}