[Deployment] Automating Building and Deployment

.github/workflows/docker-image.yml

@@ -8,37 +8,68 @@ on:
   release:
     types: [published]
 
+env:
+  PROD_BACKEND_PORT: 5001
+  DEV_BACKEND_PORT: 5003
+  PROD_FRONTEND_PORT: 5002
+  DEV_FRONTEND_PORT: 5004
+
 jobs:
   backend-build:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout backend code
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
-    - name: Build and push the Docker image for backend
-      run: |
-        if [ ${{ github.event_name }} == 'pull_request' ]; then
-          DOCKER_IMAGE_TAG=development
-        else
-          DOCKER_IMAGE_TAG=production
-        fi
-        docker build ./backend -t themanwholikestocode/archive-me-prod:backend-$DOCKER_IMAGE_TAG
-        echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin
-        docker push themanwholikestocode/archive-me-prod:backend-$DOCKER_IMAGE_TAG
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v3
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
 
+    - name: Login to DockerHub
+      run: echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin
+
+    - name: Build and push the Docker image for backend
+      uses: docker/build-push-action@v5
+      with:
+        context: ./backend
+        file: ./backend/Dockerfile
+        platforms: linux/arm64
+        push: true
+        tags: |
+          themanwholikestocode/archive-me-prod:backend-${{ github.event_name == 'pull_request' && 'development' || 'production' }}
+        build-args: |
+          CLIENT_CREDENTIALS_JSON=${{ secrets.CLIENT_CREDENTIALS_JSON }}
+          GOOGLE_CLIENT_SECRET=${{ secrets.GOOGLE_CLIENT_SECRET }}
+          GOOGLE_CLIENT_ID=${{ secrets.GOOGLE_CLIENT_ID }}
+          ENVIRONMENT=${{ github.event_name == 'pull_request' && 'dev' || 'prod' }}
+          PORT=${{ github.event_name != 'pull_request' && env.PROD_BACKEND_PORT || env.DEV_BACKEND_PORT }}
+
   frontend-build:
     runs-on: ubuntu-latest
     steps:
     - name: Checkout frontend code
       uses: actions/checkout@v3
-
+
+    - name: Set up QEMU
+      uses: docker/setup-qemu-action@v3
+
+    - name: Set up Docker Buildx
+      uses: docker/setup-buildx-action@v3
+
+    - name: Login to DockerHub
+      run: echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin
+
     - name: Build and push the Docker image for frontend
-      run: |
-        if [ ${{ github.event_name }} == 'pull_request' ]; then
-          DOCKER_IMAGE_TAG=development
-        else
-          DOCKER_IMAGE_TAG=production
-        fi
-        docker build ./frontend -t themanwholikestocode/archive-me-prod:frontend-$DOCKER_IMAGE_TAG
-        echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin
-        docker push themanwholikestocode/archive-me-prod:frontend-$DOCKER_IMAGE_TAG
+      uses: docker/build-push-action@v5
+      with:
+        context: ./frontend
+        file: ./frontend/Dockerfile
+        platforms: linux/arm64
+        push: true
+        tags: |
+          themanwholikestocode/archive-me-prod:frontend-${{ github.event_name == 'pull_request' && 'development' || 'production' }}
+        build-args: |
+          ENVIRONMENT=${{ github.event_name == 'pull_request' && 'dev' || 'prod' }}
+          PORT=${{ github.event_name != 'pull_request' && env.PROD_FRONTEND_PORT || env.DEV_FRONTEND_PORT }}

.gitignore

@@ -1,7 +1,8 @@
-
 backend/client_secrets.json
 client_secrets.json
-*.pyc
-*.pyc
 mycreds.txt
-backend/settings.yaml
+backend/.env
+frontend/.env
+*.pyc
+*.DS_Store
+backend/support/.DS_Store

backend/Dockerfile

@@ -4,10 +4,24 @@ WORKDIR /backend
 
 ADD . /backend
 
-RUN pip install -r requirements.txt
+ARG GOOGLE_CLIENT_SECRET
+ENV GOOGLE_CLIENT_SECRET=$GOOGLE_CLIENT_SECRET
+
+ARG GOOGLE_CLIENT_ID
+ENV GOOGLE_CLIENT_ID=$GOOGLE_CLIENT_ID
+
+ARG CLIENT_CREDENTIALS_JSON
+ENV CLIENT_CREDENTIALS_JSON=$CLIENT_CREDENTIALS_JSON
+RUN echo "$CLIENT_CREDENTIALS_JSON" > /backend/credentials.json
 
-EXPOSE 5001
+ARG ENVIRONMENT
+ENV ENVIRONMENT=$ENVIRONMENT
 
-CMD python ./app.py
+ARG PORT
+ENV PORT=$PORT
 
+EXPOSE $PORT
+
+RUN pip install -r requirements.txt
 
+CMD python ./app.py

backend/app.py

@@ -3,12 +3,17 @@
 import threading
 import time
 import uuid
+
+from dotenv import load_dotenv
 from flask import Flask, abort, after_this_request, jsonify, request, send_from_directory
 from flask_cors import CORS, cross_origin
 from flask_apscheduler import APScheduler
+import yaml
+
 from blackboard_scraper import BlackboardSession
 from file_management import clean_up_session_files, delete_session_files, list_files_in_drive_folder, update_drive_directory, clean_up_docs_files
 import config
+
 from pydrive2.auth import GoogleAuth
 from pydrive2.drive import GoogleDrive
 
@@ -22,6 +27,9 @@
 # Initialize Logging
 logging.basicConfig(level=logging.INFO)
 
+# Import dot env variables
+load_dotenv()
+
 
 def is_file_valid(file_path):
     return os.path.isfile(file_path) and not os.path.islink(file_path)
@@ -34,9 +42,10 @@ def remove_file_safely(file_path):
     except OSError as error:
         app.logger.error(f"Error removing file: {error}")
 
+
 @scheduler.task('interval', id='clean_up', seconds=600)
 def clean_up_and_upload_files_to_google_drive(file_path=None):
-    
+
     if file_path:
         remove_file_safely(file_path)
 
@@ -50,8 +59,34 @@ def clean_up_and_upload_files_to_google_drive(file_path=None):
 
 
 def authorize_drive():
-    gauth = GoogleAuth(settings_file='settings.yaml')
-    gauth.LocalWebserverAuth()
+    current_directory = os.getcwd()
+
+    if 'backend' in current_directory:
+        settings_path = 'settings.yaml'
+    elif 'Archive-Me' in current_directory:
+        settings_path = 'backend/settings.yaml'
+    else:
+        raise Exception("Unable to locate settings file.")
+
+    with open(settings_path, 'r') as file:
+        settings = yaml.safe_load(file)
+
+    settings['client_config']['client_id'] = os.environ.get('GOOGLE_CLIENT_ID')
+    settings['client_config']['client_secret'] = os.environ.get(
+        'GOOGLE_CLIENT_SECRET')
+
+    gauth = GoogleAuth(settings=settings)
+
+    if os.path.isfile("credentials.json"):
+        gauth.LoadCredentialsFile("credentials.json")
+    else:
+        gauth.LocalWebserverAuth()
+        gauth.SaveCredentialsFile("credentials.json")
+
+    if gauth.access_token_expired:
+        gauth.Refresh()
+        gauth.SaveCredentialsFile("credentials.json")
+
     drive = GoogleDrive(gauth)
     return drive
 
@@ -130,6 +165,7 @@ def delete_inactive_bb_sessions(inactivity_threshold_seconds=180):
 def index():
     return jsonify({'message': "Welcome to the ArchiveMe's Blackboard Scraper API"})
 
+
 @app.route('/login', methods=['POST'])
 @cross_origin()
 def login():
@@ -220,9 +256,9 @@ def list_directory(path):
     # Check if there's only one file returned
     if len(items) == 1 and items[0][3] == 'FILE':
         # Assuming 'file_id' and 'file_name' are available based on the user selection
-        file_id = items[0][2]  
-        file_name = items[0][0] 
-        
+        file_id = items[0][2]
+        file_name = items[0][0]
+
         # Update the session_files_path based on the current directory and create if it doesn't exist
         current_dir = os.path.dirname(os.path.abspath(__file__))
         if os.path.basename(current_dir) != 'backend':
@@ -235,9 +271,9 @@ def list_directory(path):
         if not os.path.exists(session_files_path):
             os.makedirs(session_files_path)
         full_path = os.path.join(session_files_path, file_name)
-        
+
         file = drive.CreateFile({'id': file_id})
-        print('Downloading file %s from Google Drive' % file_name) 
+        print('Downloading file %s from Google Drive' % file_name)
         file.GetContentFile(full_path)
 
         @after_this_request
@@ -246,9 +282,9 @@ def trigger_post_download_operations(response):
                 target=clean_up_and_upload_files_to_google_drive, args=(full_path,))
             thread.start()
             return response
-        
+
         return send_from_directory(session_files_path, file_name, as_attachment=True)
-    
+
     return jsonify(items)
 
 
@@ -258,12 +294,16 @@ def list_root_directory():
 
 
 if __name__ == '__main__':
-    
+
     drive = authorize_drive()
 
+    if not drive:
+        app.logger.error("Error authorizing Google Drive")
+        exit(1)
+
     team_drive_id = '0AFReXfsUal4rUk9PVA'
 
     scheduler.init_app(app)
     scheduler.start()
 
-    app.run(host='0.0.0.0', port=app.config['PORT'], debug=app.config['DEBUG'])
+    app.run(host='0.0.0.0', port=app.config['PORT'], debug=app.config['DEBUG'])

backend/config.py

@@ -1,6 +1,18 @@
 # config.py
-from selenium.webdriver.chrome.options import Options
-
+import os
+from dotenv import load_dotenv
+import sys
 # Flask configuration
-DEBUG = True  # Set to False in production
-PORT = 5001  # Port number for the Flask server
+load_dotenv()
+
+env = os.environ.get('ENVIRONMENT')
+
+if env == 'dev':
+    PORT = 5003
+    DEBUG = True
+elif env == 'prod':
+    PORT = 5001
+    DEBUG = False
+else:
+    print("Environment not specified. Please provide a valid environment.")
+    sys.exit(1)

backend/requirements.txt

@@ -7,3 +7,4 @@ gunicorn
 flask_cors
 flask_apscheduler
 pydrive2
+python-dotenv

backend/settings.yaml

@@ -0,0 +1,13 @@
+client_config_backend: settings
+client_config:
+  client_id: ${GOOGLE_CLIENT_ID}
+  client_secret: ${GOOGLE_CLIENT_SECRET}
+
+save_credentials: True
+save_credentials_backend: file
+save_credentials_file: "credentials.json"
+
+get_refresh_token: True
+
+oauth_scope:
+  - "https://www.googleapis.com/auth/drive"

frontend/Dockerfile

@@ -6,6 +6,12 @@ ADD . /frontend
 
 RUN pip install -r requirements.txt
 
-EXPOSE 5002
+ARG ENVIRONMENT
+ENV ENVIRONMENT=$ENVIRONMENT
+
+ARG PORT
+ENV PORT=$PORT
+
+EXPOSE $PORT
 
 CMD python ./app.py

frontend/app.py

@@ -1,4 +1,4 @@
-from flask import Flask, render_template, request, jsonify, send_from_directory, abort
+from flask import Flask, render_template
 from flask_cors import CORS, cross_origin
 import os
 import logging
@@ -28,11 +28,6 @@ def demo():
 def directory():
     return render_template('directory.html')
 
-# Add a login route for demonstration
-@app.route('/login', methods=['POST'])
-def login():
-    # Your login logic here
-    return jsonify(success=True)
 
 if __name__ == '__main__':
     app.run(host='0.0.0.0', port=app.config['PORT'], debug=app.config['DEBUG'])

frontend/config.json

@@ -0,0 +1,6 @@
+{
+    "apiUrl":{
+        "dev":"http://localhost:5003",
+        "prod":"https://api.archive-me.net"
+    }
+}

frontend/config.py

@@ -1,12 +1,22 @@
 # config.py
+import os
+from dotenv import load_dotenv
+import sys
 
 # Flask configuration
-DEBUG = True  # Set to False in production
-PORT = 5002  # Port number for the Flask server
+load_dotenv()
+
+env = os.environ.get('ENVIRONMENT')
+
+if env == 'dev':
+    PORT = 5004
+    DEBUG = True
+elif env == 'prod':
+    PORT = 5002
+    DEBUG = False
+else:
+    print("Environment not specified. Please provide a valid environment.")
+    sys.exit(1)
 
 # CORS Configurations
 CORS_HEADERS = 'Content-Type'
-
-# Security configurations
-# It's advisable to use environment variables for sensitive data
-# Example: SECRET_KEY = os.environ.get('SECRET_KEY') or 'a-default-secret'

frontend/requirements.txt

@@ -5,4 +5,5 @@ ray
 argparse
 bs4
 gunicorn
-flask_cors
+flask_cors
+python-dotenv