From 74bc4f027c84e213d1242b85efcf44575b57936a Mon Sep 17 00:00:00 2001 From: leej3 Date: Tue, 3 Dec 2024 15:49:57 +0000 Subject: [PATCH] configure deployment --- .github/workflows/deploy-docker.yml | 2 +- web/.env.template | 1 + web/deploy/docker-compose.yaml | 53 +++++++------------ web/js_dashboard/Dockerfile | 10 ++++ web/js_dashboard/backend/main.py | 15 ------ .../backend/services/data_service.py | 42 --------------- web/js_dashboard/src/services/api.ts | 4 +- web/js_dashboard_backend/Dockerfile | 11 ++++ web/js_dashboard_backend/main.py | 18 +++++++ .../routers/metrics.py | 6 ++- .../services/data_service.py | 42 +++++++++++++++ 11 files changed, 108 insertions(+), 96 deletions(-) create mode 100644 web/js_dashboard/Dockerfile delete mode 100644 web/js_dashboard/backend/main.py delete mode 100644 web/js_dashboard/backend/services/data_service.py create mode 100644 web/js_dashboard_backend/Dockerfile create mode 100644 web/js_dashboard_backend/main.py rename web/{js_dashboard/backend => js_dashboard_backend}/routers/metrics.py (90%) create mode 100644 web/js_dashboard_backend/services/data_service.py diff --git a/.github/workflows/deploy-docker.yml b/.github/workflows/deploy-docker.yml index 72889d18..45d5d246 100644 --- a/.github/workflows/deploy-docker.yml +++ b/.github/workflows/deploy-docker.yml @@ -75,7 +75,7 @@ jobs: needs: [infrastructure-modified, set-development-environment] strategy: matrix: - deployment: ["api", "dashboard"] + deployment: ["js_dashboard", "js_dashboard_backend"] uses: ./.github/workflows/build-docker.yml if: ( (github.event_name == 'pull_request' && github.event.pull_request.merged == true && needs.infrastructure-modified.outputs.modified-files != 'true' && needs.set-development-environment.result == 'success') || (github.event_name == 'workflow_dispatch') || (github.event_name == 'workflow_call') ) secrets: inherit diff --git a/web/.env.template b/web/.env.template index 8539ff6f..6954e1e4 100644 --- a/web/.env.template +++ b/web/.env.template @@ -9,3 +9,4 @@ # DASHBOARD_IMAGE_TAG="/osm_dashboard:" # MONGODB_URI="your-mongodb-uri" # SSH_KEY_PATH=/home/user/.ssh/... # Path to the SSH key used to access the EC2 instance +# VITE_API_BASE_URL=http://localhost:8000/api # Base URL for the API diff --git a/web/deploy/docker-compose.yaml b/web/deploy/docker-compose.yaml index 9c8fae07..c2198e66 100644 --- a/web/deploy/docker-compose.yaml +++ b/web/deploy/docker-compose.yaml @@ -1,59 +1,42 @@ name: osm services: - web_api: + js_dashboard_backend: image: "${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/api-shared:${RELEASE_TAG}" pull_policy: always - environment: - MONGODB_URI: ${MONGODB_URI} - working_dir: /app/app + restart: always expose: - - "80" - labels: - - traefik.enable=true - - traefik.docker.network=osm_traefik-public - - traefik.http.routers.osm_web_api.rule=Host(`${DEPLOYMENT_URI}`) && PathPrefix(`/api`) - - "traefik.http.middlewares.strip-api-prefix.stripprefix.prefixes=/api" - - "traefik.http.routers.osm_web_api.middlewares=strip-api-prefix@docker" - - "traefik.http.routers.osm_web_api.entrypoints=web,websecure" - - "traefik.http.routers.osm_web_api.service=osm_web_api" - - traefik.http.services.osm_web_api.loadbalancer.server.port=80 - - traefik.http.routers.osm_web_api.tls=true - - traefik.http.routers.osm_web_api.tls.certresolver=le - - "traefik.http.routers.osm_web_api.priority=100" # Higher priority than dashboard + - "8000" networks: - traefik-public - restart: always - healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:80/health"] - interval: 10s - timeout: 5s - retries: 3 - start_period: 30s deploy: update_config: order: start-first failure_action: rollback delay: 10s - dashboard: - image: "${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/dashboard-shared:${RELEASE_TAG}" + js_dashboard: + image: "${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_REGION}.amazonaws.com/js_dashboard-shared:${RELEASE_TAG}" pull_policy: always + restart: always environment: - MONGODB_URI: ${MONGODB_URI} - working_dir: /app + - VITE_API_BASE_URL=http://js_dashboard_backend:8000/api labels: - traefik.enable=true - traefik.docker.network=osm_traefik-public - - traefik.http.routers.dashboard.rule=Host(`${DEPLOYMENT_URI}`) - - traefik.http.routers.dashboard.entrypoints=web,websecure - - traefik.http.services.dashboard.loadbalancer.server.port=8501 - - traefik.http.routers.dashboard.tls=true - - traefik.http.routers.dashboard.tls.certresolver=le + - traefik.http.routers.js_dashboard.rule=Host(`${DEPLOYMENT_URI}`) + - traefik.http.routers.js_dashboard.entrypoints=web,websecure + - traefik.http.services.js_dashboard.loadbalancer.server.port=5173 + - traefik.http.routers.js_dashboard.tls=true + - traefik.http.routers.js_dashboard.tls.certresolver=le expose: - - "8501" + - "5173" networks: - traefik-public - restart: always + deploy: + update_config: + order: start-first + failure_action: rollback + delay: 10s reverse_proxy: image: traefik diff --git a/web/js_dashboard/Dockerfile b/web/js_dashboard/Dockerfile new file mode 100644 index 00000000..9b497fa8 --- /dev/null +++ b/web/js_dashboard/Dockerfile @@ -0,0 +1,10 @@ +FROM node:18-alpine as build + +WORKDIR /app + +COPY web/js_dashboard/ /app + +RUN npm install +EXPOSE 5173 + +CMD ["npm", "run", "dev","--","--host"] diff --git a/web/js_dashboard/backend/main.py b/web/js_dashboard/backend/main.py deleted file mode 100644 index 48906c25..00000000 --- a/web/js_dashboard/backend/main.py +++ /dev/null @@ -1,15 +0,0 @@ -from fastapi import FastAPI -from fastapi.middleware.cors import CORSMiddleware -from routers import metrics - -app = FastAPI(title="OpenSciMetrics API") - -app.add_middleware( - CORSMiddleware, - allow_origins=["http://localhost:5173"], - allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], -) - -app.include_router(metrics.router, prefix="/api") \ No newline at end of file diff --git a/web/js_dashboard/backend/services/data_service.py b/web/js_dashboard/backend/services/data_service.py deleted file mode 100644 index 87727112..00000000 --- a/web/js_dashboard/backend/services/data_service.py +++ /dev/null @@ -1,42 +0,0 @@ -import pandas as pd -from typing import List, Dict - -class DataService: - def __init__(self): - self.df = pd.read_parquet("matches.parquet") - - def get_summary(self) -> Dict: - return { - "totalRecords": len(self.df), - "openCodePercentage": float(self.df['is_open_code'].mean() * 100), - "openDataPercentage": float(self.df['is_open_data'].mean() * 100), - "uniqueJournals": int(self.df['journal'].nunique()), - "uniqueCountries": int(self.df['affiliation_country'].nunique()), - } - - def get_timeseries(self) -> List[Dict]: - yearly_stats = self.df.groupby('year').agg({ - 'is_open_code': 'mean', - 'is_open_data': 'mean' - }).reset_index() - - return [ - { - "year": int(row['year']), - "openCode": round(float(row['is_open_code'] * 100), 2), - "openData": round(float(row['is_open_data'] * 100), 2) - } - for _, row in yearly_stats.iterrows() - ] - - def get_country_distribution(self) -> List[Dict]: - return (self.df['affiliation_country'] - .value_counts() - .head(10) - .to_dict()) - - def get_journal_distribution(self) -> List[Dict]: - return (self.df['journal'] - .value_counts() - .head(10) - .to_dict()) \ No newline at end of file diff --git a/web/js_dashboard/src/services/api.ts b/web/js_dashboard/src/services/api.ts index 58cf6c49..45e1c8a2 100644 --- a/web/js_dashboard/src/services/api.ts +++ b/web/js_dashboard/src/services/api.ts @@ -1,7 +1,7 @@ import axios from 'axios'; import type { MetricsSummary } from '../types/metrics'; -const API_BASE_URL = 'http://localhost:8000/api'; +const API_BASE_URL = import.meta.env.VITE_API_BASE_URL || 'http://localhost:8000/api'; export const api = { async getSummary(): Promise { @@ -23,4 +23,4 @@ export const api = { const { data } = await axios.get(`${API_BASE_URL}/journal-distribution`); return Object.entries(data).map(([name, value]) => ({ name, value })); } -}; \ No newline at end of file +}; diff --git a/web/js_dashboard_backend/Dockerfile b/web/js_dashboard_backend/Dockerfile new file mode 100644 index 00000000..973abe80 --- /dev/null +++ b/web/js_dashboard_backend/Dockerfile @@ -0,0 +1,11 @@ +FROM python:3.9-slim + +WORKDIR /app +RUN pip install fastapi uvicorn pandas pyarrow + +COPY ./web/js_dashboard_backend/ /app + +COPY ./dashboard_data/matches.parquet /opt/data/matches.parquet +EXPOSE 8000 + +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/web/js_dashboard_backend/main.py b/web/js_dashboard_backend/main.py new file mode 100644 index 00000000..04315fa7 --- /dev/null +++ b/web/js_dashboard_backend/main.py @@ -0,0 +1,18 @@ +from fastapi import FastAPI +from fastapi.middleware.cors import CORSMiddleware +from routers import metrics + +app = FastAPI(title="OpenSciMetrics API") + +# Regex for localhost, IPv4 loopback, and Docker CIDR block (e.g., 192.168.0.0/16) +allowed_origin_regex = r"^(http://)?(localhost|js_dashboard_backend|127\.0\.0\.1|192\.168\.\d{1,3}\.\d{1,3}):\d+$" + +app.add_middleware( + CORSMiddleware, + allow_origin_regex=allowed_origin_regex, + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], +) + +app.include_router(metrics.router, prefix="/api") diff --git a/web/js_dashboard/backend/routers/metrics.py b/web/js_dashboard_backend/routers/metrics.py similarity index 90% rename from web/js_dashboard/backend/routers/metrics.py rename to web/js_dashboard_backend/routers/metrics.py index 49f0ca28..5611da95 100644 --- a/web/js_dashboard/backend/routers/metrics.py +++ b/web/js_dashboard_backend/routers/metrics.py @@ -4,18 +4,22 @@ router = APIRouter() data_service = DataService() + @router.get("/summary") async def get_summary(): return data_service.get_summary() + @router.get("/timeseries") async def get_timeseries(): return data_service.get_timeseries() + @router.get("/country-distribution") async def get_country_distribution(): return data_service.get_country_distribution() + @router.get("/journal-distribution") async def get_journal_distribution(): - return data_service.get_journal_distribution() \ No newline at end of file + return data_service.get_journal_distribution() diff --git a/web/js_dashboard_backend/services/data_service.py b/web/js_dashboard_backend/services/data_service.py new file mode 100644 index 00000000..3c8da407 --- /dev/null +++ b/web/js_dashboard_backend/services/data_service.py @@ -0,0 +1,42 @@ +import os +from typing import Dict, List + +import pandas as pd + +DATA_PATH = os.getenv("DATA_PATH", "/opt/data/matches.parquet") + + +class DataService: + def __init__(self): + self.df = pd.read_parquet(DATA_PATH) + + def get_summary(self) -> Dict: + return { + "totalRecords": len(self.df), + "openCodePercentage": float(self.df["is_open_code"].mean() * 100), + "openDataPercentage": float(self.df["is_open_data"].mean() * 100), + "uniqueJournals": int(self.df["journal"].nunique()), + "uniqueCountries": int(self.df["affiliation_country"].nunique()), + } + + def get_timeseries(self) -> List[Dict]: + yearly_stats = ( + self.df.groupby("year") + .agg({"is_open_code": "mean", "is_open_data": "mean"}) + .reset_index() + ) + + return [ + { + "year": int(row["year"]), + "openCode": round(float(row["is_open_code"] * 100), 2), + "openData": round(float(row["is_open_data"] * 100), 2), + } + for _, row in yearly_stats.iterrows() + ] + + def get_country_distribution(self) -> List[Dict]: + return self.df["affiliation_country"].value_counts().head(10).to_dict() + + def get_journal_distribution(self) -> List[Dict]: + return self.df["journal"].value_counts().head(10).to_dict()