diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml
index d116444..11dd3d0 100644
--- a/.github/workflows/docker-image.yml
+++ b/.github/workflows/docker-image.yml
@@ -8,37 +8,68 @@ on:
release:
types: [published]
+env:
+ PROD_BACKEND_PORT: 5001
+ DEV_BACKEND_PORT: 5003
+ PROD_FRONTEND_PORT: 5002
+ DEV_FRONTEND_PORT: 5004
+
jobs:
backend-build:
runs-on: ubuntu-latest
steps:
- name: Checkout backend code
- uses: actions/checkout@v3
+ uses: actions/checkout@v4
- - name: Build and push the Docker image for backend
- run: |
- if [ ${{ github.event_name }} == 'pull_request' ]; then
- DOCKER_IMAGE_TAG=development
- else
- DOCKER_IMAGE_TAG=production
- fi
- docker build ./backend -t themanwholikestocode/archive-me-prod:backend-$DOCKER_IMAGE_TAG
- echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin
- docker push themanwholikestocode/archive-me-prod:backend-$DOCKER_IMAGE_TAG
+ - name: Set up QEMU
+ uses: docker/setup-qemu-action@v3
+
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v3
+ - name: Login to DockerHub
+ run: echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin
+
+ - name: Build and push the Docker image for backend
+ uses: docker/build-push-action@v5
+ with:
+ context: ./backend
+ file: ./backend/Dockerfile
+ platforms: linux/arm64
+ push: true
+ tags: |
+ themanwholikestocode/archive-me-prod:backend-${{ github.event_name == 'pull_request' && 'development' || 'production' }}
+ build-args: |
+ CLIENT_CREDENTIALS_JSON=${{ secrets.CLIENT_CREDENTIALS_JSON }}
+ GOOGLE_CLIENT_SECRET=${{ secrets.GOOGLE_CLIENT_SECRET }}
+ GOOGLE_CLIENT_ID=${{ secrets.GOOGLE_CLIENT_ID }}
+ ENVIRONMENT=${{ github.event_name == 'pull_request' && 'dev' || 'prod' }}
+ PORT=${{ github.event_name != 'pull_request' && env.PROD_BACKEND_PORT || env.DEV_BACKEND_PORT }}
+
frontend-build:
runs-on: ubuntu-latest
steps:
- name: Checkout frontend code
uses: actions/checkout@v3
-
+
+ - name: Set up QEMU
+ uses: docker/setup-qemu-action@v3
+
+ - name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v3
+
+ - name: Login to DockerHub
+ run: echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin
+
- name: Build and push the Docker image for frontend
- run: |
- if [ ${{ github.event_name }} == 'pull_request' ]; then
- DOCKER_IMAGE_TAG=development
- else
- DOCKER_IMAGE_TAG=production
- fi
- docker build ./frontend -t themanwholikestocode/archive-me-prod:frontend-$DOCKER_IMAGE_TAG
- echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin
- docker push themanwholikestocode/archive-me-prod:frontend-$DOCKER_IMAGE_TAG
+ uses: docker/build-push-action@v5
+ with:
+ context: ./frontend
+ file: ./frontend/Dockerfile
+ platforms: linux/arm64
+ push: true
+ tags: |
+ themanwholikestocode/archive-me-prod:frontend-${{ github.event_name == 'pull_request' && 'development' || 'production' }}
+ build-args: |
+ ENVIRONMENT=${{ github.event_name == 'pull_request' && 'dev' || 'prod' }}
+ PORT=${{ github.event_name != 'pull_request' && env.PROD_FRONTEND_PORT || env.DEV_FRONTEND_PORT }}
diff --git a/.gitignore b/.gitignore
index eaf1d5d..78a3d75 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,7 +1,8 @@
-
backend/client_secrets.json
client_secrets.json
-*.pyc
-*.pyc
mycreds.txt
-backend/settings.yaml
+backend/.env
+frontend/.env
+*.pyc
+*.DS_Store
+backend/support/.DS_Store
diff --git a/backend/Dockerfile b/backend/Dockerfile
index 397a86a..052669d 100644
--- a/backend/Dockerfile
+++ b/backend/Dockerfile
@@ -4,10 +4,24 @@ WORKDIR /backend
ADD . /backend
-RUN pip install -r requirements.txt
+ARG GOOGLE_CLIENT_SECRET
+ENV GOOGLE_CLIENT_SECRET=$GOOGLE_CLIENT_SECRET
+
+ARG GOOGLE_CLIENT_ID
+ENV GOOGLE_CLIENT_ID=$GOOGLE_CLIENT_ID
+
+ARG CLIENT_CREDENTIALS_JSON
+ENV CLIENT_CREDENTIALS_JSON=$CLIENT_CREDENTIALS_JSON
+RUN echo "$CLIENT_CREDENTIALS_JSON" > /backend/credentials.json
-EXPOSE 5001
+ARG ENVIRONMENT
+ENV ENVIRONMENT=$ENVIRONMENT
-CMD python ./app.py
+ARG PORT
+ENV PORT=$PORT
+EXPOSE $PORT
+
+RUN pip install -r requirements.txt
+CMD python ./app.py
\ No newline at end of file
diff --git a/backend/__pycache__/app.cpython-311.pyc b/backend/__pycache__/app.cpython-311.pyc
deleted file mode 100644
index 19efd5b..0000000
Binary files a/backend/__pycache__/app.cpython-311.pyc and /dev/null differ
diff --git a/backend/__pycache__/blackboard_scraper.cpython-311.pyc b/backend/__pycache__/blackboard_scraper.cpython-311.pyc
deleted file mode 100644
index 6ea5ea0..0000000
Binary files a/backend/__pycache__/blackboard_scraper.cpython-311.pyc and /dev/null differ
diff --git a/backend/__pycache__/config.cpython-311.pyc b/backend/__pycache__/config.cpython-311.pyc
deleted file mode 100644
index 61d96a6..0000000
Binary files a/backend/__pycache__/config.cpython-311.pyc and /dev/null differ
diff --git a/backend/__pycache__/file_management.cpython-311.pyc b/backend/__pycache__/file_management.cpython-311.pyc
deleted file mode 100644
index f22f580..0000000
Binary files a/backend/__pycache__/file_management.cpython-311.pyc and /dev/null differ
diff --git a/backend/__pycache__/pdf_compressor.cpython-311.pyc b/backend/__pycache__/pdf_compressor.cpython-311.pyc
deleted file mode 100644
index 4f616e2..0000000
Binary files a/backend/__pycache__/pdf_compressor.cpython-311.pyc and /dev/null differ
diff --git a/backend/app.py b/backend/app.py
index a464246..9a352b4 100644
--- a/backend/app.py
+++ b/backend/app.py
@@ -3,12 +3,17 @@
import threading
import time
import uuid
+
+from dotenv import load_dotenv
from flask import Flask, abort, after_this_request, jsonify, request, send_from_directory
from flask_cors import CORS, cross_origin
from flask_apscheduler import APScheduler
+import yaml
+
from blackboard_scraper import BlackboardSession
from file_management import clean_up_session_files, delete_session_files, list_files_in_drive_folder, update_drive_directory, clean_up_docs_files
import config
+
from pydrive2.auth import GoogleAuth
from pydrive2.drive import GoogleDrive
@@ -22,6 +27,9 @@
# Initialize Logging
logging.basicConfig(level=logging.INFO)
+# Import dot env variables
+load_dotenv()
+
def is_file_valid(file_path):
return os.path.isfile(file_path) and not os.path.islink(file_path)
@@ -34,9 +42,10 @@ def remove_file_safely(file_path):
except OSError as error:
app.logger.error(f"Error removing file: {error}")
+
@scheduler.task('interval', id='clean_up', seconds=600)
def clean_up_and_upload_files_to_google_drive(file_path=None):
-
+
if file_path:
remove_file_safely(file_path)
@@ -50,8 +59,34 @@ def clean_up_and_upload_files_to_google_drive(file_path=None):
def authorize_drive():
- gauth = GoogleAuth(settings_file='settings.yaml')
- gauth.LocalWebserverAuth()
+ current_directory = os.getcwd()
+
+ if 'backend' in current_directory:
+ settings_path = 'settings.yaml'
+ elif 'Archive-Me' in current_directory:
+ settings_path = 'backend/settings.yaml'
+ else:
+ raise Exception("Unable to locate settings file.")
+
+ with open(settings_path, 'r') as file:
+ settings = yaml.safe_load(file)
+
+ settings['client_config']['client_id'] = os.environ.get('GOOGLE_CLIENT_ID')
+ settings['client_config']['client_secret'] = os.environ.get(
+ 'GOOGLE_CLIENT_SECRET')
+
+ gauth = GoogleAuth(settings=settings)
+
+ if os.path.isfile("credentials.json"):
+ gauth.LoadCredentialsFile("credentials.json")
+ else:
+ gauth.LocalWebserverAuth()
+ gauth.SaveCredentialsFile("credentials.json")
+
+ if gauth.access_token_expired:
+ gauth.Refresh()
+ gauth.SaveCredentialsFile("credentials.json")
+
drive = GoogleDrive(gauth)
return drive
@@ -130,6 +165,7 @@ def delete_inactive_bb_sessions(inactivity_threshold_seconds=180):
def index():
return jsonify({'message': "Welcome to the ArchiveMe's Blackboard Scraper API"})
+
@app.route('/login', methods=['POST'])
@cross_origin()
def login():
@@ -220,9 +256,9 @@ def list_directory(path):
# Check if there's only one file returned
if len(items) == 1 and items[0][3] == 'FILE':
# Assuming 'file_id' and 'file_name' are available based on the user selection
- file_id = items[0][2]
- file_name = items[0][0]
-
+ file_id = items[0][2]
+ file_name = items[0][0]
+
# Update the session_files_path based on the current directory and create if it doesn't exist
current_dir = os.path.dirname(os.path.abspath(__file__))
if os.path.basename(current_dir) != 'backend':
@@ -235,9 +271,9 @@ def list_directory(path):
if not os.path.exists(session_files_path):
os.makedirs(session_files_path)
full_path = os.path.join(session_files_path, file_name)
-
+
file = drive.CreateFile({'id': file_id})
- print('Downloading file %s from Google Drive' % file_name)
+ print('Downloading file %s from Google Drive' % file_name)
file.GetContentFile(full_path)
@after_this_request
@@ -246,9 +282,9 @@ def trigger_post_download_operations(response):
target=clean_up_and_upload_files_to_google_drive, args=(full_path,))
thread.start()
return response
-
+
return send_from_directory(session_files_path, file_name, as_attachment=True)
-
+
return jsonify(items)
@@ -258,12 +294,16 @@ def list_root_directory():
if __name__ == '__main__':
-
+
drive = authorize_drive()
+ if not drive:
+ app.logger.error("Error authorizing Google Drive")
+ exit(1)
+
team_drive_id = '0AFReXfsUal4rUk9PVA'
scheduler.init_app(app)
scheduler.start()
- app.run(host='0.0.0.0', port=app.config['PORT'], debug=app.config['DEBUG'])
\ No newline at end of file
+ app.run(host='0.0.0.0', port=app.config['PORT'], debug=app.config['DEBUG'])
diff --git a/backend/config.py b/backend/config.py
index 802de4d..c1dde59 100644
--- a/backend/config.py
+++ b/backend/config.py
@@ -1,6 +1,18 @@
# config.py
-from selenium.webdriver.chrome.options import Options
-
+import os
+from dotenv import load_dotenv
+import sys
# Flask configuration
-DEBUG = True # Set to False in production
-PORT = 5001 # Port number for the Flask server
\ No newline at end of file
+load_dotenv()
+
+env = os.environ.get('ENVIRONMENT')
+
+if env == 'dev':
+ PORT = 5003
+ DEBUG = True
+elif env == 'prod':
+ PORT = 5001
+ DEBUG = False
+else:
+ print("Environment not specified. Please provide a valid environment.")
+ sys.exit(1)
diff --git a/backend/requirements.txt b/backend/requirements.txt
index d5177b0..3605f18 100644
--- a/backend/requirements.txt
+++ b/backend/requirements.txt
@@ -7,3 +7,4 @@ gunicorn
flask_cors
flask_apscheduler
pydrive2
+python-dotenv
\ No newline at end of file
diff --git a/backend/settings.yaml b/backend/settings.yaml
new file mode 100644
index 0000000..df67398
--- /dev/null
+++ b/backend/settings.yaml
@@ -0,0 +1,13 @@
+client_config_backend: settings
+client_config:
+ client_id: ${GOOGLE_CLIENT_ID}
+ client_secret: ${GOOGLE_CLIENT_SECRET}
+
+save_credentials: True
+save_credentials_backend: file
+save_credentials_file: "credentials.json"
+
+get_refresh_token: True
+
+oauth_scope:
+ - "https://www.googleapis.com/auth/drive"
diff --git a/frontend/Dockerfile b/frontend/Dockerfile
index 05a97d9..8900a26 100644
--- a/frontend/Dockerfile
+++ b/frontend/Dockerfile
@@ -6,6 +6,12 @@ ADD . /frontend
RUN pip install -r requirements.txt
-EXPOSE 5002
+ARG ENVIRONMENT
+ENV ENVIRONMENT=$ENVIRONMENT
+
+ARG PORT
+ENV PORT=$PORT
+
+EXPOSE $PORT
CMD python ./app.py
\ No newline at end of file
diff --git a/frontend/__pycache__/app.cpython-311.pyc b/frontend/__pycache__/app.cpython-311.pyc
deleted file mode 100644
index 4db6fb9..0000000
Binary files a/frontend/__pycache__/app.cpython-311.pyc and /dev/null differ
diff --git a/frontend/__pycache__/config.cpython-311.pyc b/frontend/__pycache__/config.cpython-311.pyc
deleted file mode 100644
index 6f6366c..0000000
Binary files a/frontend/__pycache__/config.cpython-311.pyc and /dev/null differ
diff --git a/frontend/__pycache__/wsgi.cpython-311.pyc b/frontend/__pycache__/wsgi.cpython-311.pyc
deleted file mode 100644
index a8854af..0000000
Binary files a/frontend/__pycache__/wsgi.cpython-311.pyc and /dev/null differ
diff --git a/frontend/app.py b/frontend/app.py
index 9317831..cfbfe95 100644
--- a/frontend/app.py
+++ b/frontend/app.py
@@ -1,4 +1,4 @@
-from flask import Flask, render_template, request, jsonify, send_from_directory, abort
+from flask import Flask, render_template
from flask_cors import CORS, cross_origin
import os
import logging
@@ -28,11 +28,6 @@ def demo():
def directory():
return render_template('directory.html')
-# Add a login route for demonstration
-@app.route('/login', methods=['POST'])
-def login():
- # Your login logic here
- return jsonify(success=True)
if __name__ == '__main__':
app.run(host='0.0.0.0', port=app.config['PORT'], debug=app.config['DEBUG'])
diff --git a/frontend/config.json b/frontend/config.json
new file mode 100644
index 0000000..0e03787
--- /dev/null
+++ b/frontend/config.json
@@ -0,0 +1,6 @@
+{
+ "apiUrl":{
+ "dev":"http://localhost:5003",
+ "prod":"https://api.archive-me.net"
+ }
+}
\ No newline at end of file
diff --git a/frontend/config.py b/frontend/config.py
index 26fd0a1..de0cb9f 100644
--- a/frontend/config.py
+++ b/frontend/config.py
@@ -1,12 +1,22 @@
# config.py
+import os
+from dotenv import load_dotenv
+import sys
# Flask configuration
-DEBUG = True # Set to False in production
-PORT = 5002 # Port number for the Flask server
+load_dotenv()
+
+env = os.environ.get('ENVIRONMENT')
+
+if env == 'dev':
+ PORT = 5004
+ DEBUG = True
+elif env == 'prod':
+ PORT = 5002
+ DEBUG = False
+else:
+ print("Environment not specified. Please provide a valid environment.")
+ sys.exit(1)
# CORS Configurations
CORS_HEADERS = 'Content-Type'
-
-# Security configurations
-# It's advisable to use environment variables for sensitive data
-# Example: SECRET_KEY = os.environ.get('SECRET_KEY') or 'a-default-secret'
diff --git a/frontend/requirements.txt b/frontend/requirements.txt
index 5c38da7..cd26aff 100644
--- a/frontend/requirements.txt
+++ b/frontend/requirements.txt
@@ -5,4 +5,5 @@ ray
argparse
bs4
gunicorn
-flask_cors
\ No newline at end of file
+flask_cors
+python-dotenv
\ No newline at end of file
diff --git a/frontend/static/helpers.js b/frontend/static/helpers.js
new file mode 100644
index 0000000..1e31f32
--- /dev/null
+++ b/frontend/static/helpers.js
@@ -0,0 +1,27 @@
+function getConfig() {
+ const config = {
+ apiUrl: {
+ dev: "http://localhost:5003",
+ prod: "https://api.archive-me.net"
+ }
+ };
+ return config;
+}
+
+function getEnv() {
+ const hostname = window.location.hostname;
+ return hostname.includes('localhost') ? 'dev' : 'prod';
+}
+
+function getApiUrl() {
+ const config = getConfig();
+ const env = getEnv();
+
+ if (!config.apiUrl[env]) {
+ throw new Error('Api url not found.');
+ }
+
+ return config.apiUrl[env];
+}
+
+export { getEnv, getApiUrl };
diff --git a/frontend/static/scripts.js b/frontend/static/scripts.js
index a619196..b0bc296 100755
--- a/frontend/static/scripts.js
+++ b/frontend/static/scripts.js
@@ -1,3 +1,5 @@
+import { getApiUrl } from './helpers.js';
+
$(function () {
// init feather icons
feather.replace();
@@ -83,7 +85,8 @@ $(function () {
const app = (() => {
let fileKeyGlobal = null;
let currentPath = '';
- const apiUrl = 'https://api.archive-me.net';
+
+ const apiUrl = getApiUrl();
const showLoadingScreen = () => {
const loadingScreen = document.getElementById("loading-screen");
diff --git a/frontend/templates/demo.html b/frontend/templates/demo.html
index 75cb4d6..ed4bdfc 100644
--- a/frontend/templates/demo.html
+++ b/frontend/templates/demo.html
@@ -108,7 +108,7 @@ About Archive Me
-
+