Merge pull request #7 from ISSUIUC/AV-1045-web-server-development

Av 1045 web server development

.gitignore

@@ -1,5 +1,9 @@
-remote/
-post_return.txt
-__pycache__/
-node_modules/
-.env
+remote/
+post_return.txt
+__pycache__/
+node_modules/
+**/.DS_Store
+# TODO: Ignore this
+# secrets/
+node_modules/
+.env

Central-Server/API/.env

@@ -0,0 +1,3 @@
+BROWSER=none
+REACT_APP_GITHUB_CLIENT_ID = '70662875937be8f7bce6'
+REACT_APP_GITHUB_CLIENT_SECRET = '89c565b3761740ca3710c3c31d41c1859fa3ab95'

Central-Server/API/Dockerfile

@@ -1,7 +1,8 @@
-FROM python:3.8-slim-buster
-WORKDIR /src/api
-COPY requirements.txt requirements.txt
-RUN pip3 install -r requirements.txt
-COPY . .
-EXPOSE 443
+FROM python:3.8-slim-buster
+WORKDIR /src/api
+RUN apt-get update && apt-get install -y libpq-dev gcc
+COPY requirements.txt requirements.txt
+RUN pip3 install -r requirements.txt
+COPY . .
+EXPOSE 443
 CMD ["python3", "main.py", "dev"]

Central-Server/API/Dockerfile.prod

@@ -1,7 +1,8 @@
-FROM python:3.8-slim-buster
-WORKDIR /src/api
-COPY requirements.txt requirements.txt
-RUN pip3 install -r requirements.txt
-COPY . .
-EXPOSE 443
-CMD ["python3", "main.py", "prod"]
+FROM python:3.8-slim-buster
+WORKDIR /src/api
+RUN apt-get update && apt-get install -y libpq-dev gcc
+COPY requirements.txt requirements.txt
+RUN pip3 install -r requirements.txt
+COPY . .
+EXPOSE 443
+CMD ["python3", "main.py", "prod"]

Central-Server/API/blueprints/jobs_blueprint.py

@@ -0,0 +1,159 @@
+import os
+import os.path
+
+from apiflask import APIBlueprint
+from flask import abort, jsonify, request, Response
+
+import internal.database as database
+import internal.auth as auth
+import internal.sanitizers as sanitizers
+from internal.jobs import *
+
+
+def sanitize_job_info(job):
+    del job["output_path"]
+    job["run_status"] = JobStatus(job["run_status"]).name
+    return job
+
+
+jobs_blueprint = APIBlueprint('jobs', __name__)
+
+JOB_OUTPUT_DIR = "output/"
+JOB_OUTPUT_PREFIX = JOB_OUTPUT_DIR + "job_"
+
+
+@jobs_blueprint.route('/jobs', methods=["GET"])
+def list_jobs():
+    """
+    List all the latest 10 jobs
+
+    Additional parameters:
+    size: Size of page
+    page: the page
+
+    This will return an empty array if there is no available job for that page.
+    """
+    if not (auth.authenticate_request(request)):
+        abort(403)
+    # List out all the jobs in the database
+    size = request.args.get("size", default=10, type=str)
+    page = request.args.get("page", default=0, type=int)
+    conn = database.connect()
+    cursor = conn.cursor()
+    cursor.execute(
+        "SELECT * FROM hilsim_runs ORDER BY run_status DESC limit %s offset %s",
+        (size, page * size))
+    # Sort through the json and set status
+    structs = database.convert_database_list(cursor, cursor.fetchall())
+    structs = [job._asdict() for job in structs]
+    for job in structs:
+        # Additional formatting
+        sanitize_job_info(job)
+
+    return jsonify(structs), 200
+
+
+@jobs_blueprint.route('/job/<int:job_id>', methods=["GET"])
+@jobs_blueprint.output(JobOutSchema())
+def job_information(job_id):
+    """
+    Gets the details of a job
+    """
+    # Get the jobs data from the job id
+    if (auth.authenticate_request(request) == False):
+        abort(403)
+    # List out all the jobs in the database
+    conn = database.connect()
+    cursor = conn.cursor()
+    cursor.execute("SELECT * FROM hilsim_runs where run_id=%s", (job_id,))
+    data = cursor.fetchone()
+    if data is None:
+        return jsonify({"error": "Job not found"}), 404
+    return sanitize_job_info(
+        database.convert_database_tuple(
+            cursor, data)._asdict())
+
+
+@jobs_blueprint.route('/job/<int:job_id>/data', methods=["GET"])
+def job_data(job_id):
+    """
+    Gets the results of a job
+    """
+    # Get the jobs data from the job id
+    if (auth.authenticate_request(request) == False):
+        abort(403)
+    # List out all the jobs in the database
+    conn = database.connect()
+    cursor = conn.cursor()
+    cursor.execute("SELECT * FROM hilsim_runs where run_id=%s", (job_id,))
+    data = cursor.fetchone()
+    data = database.convert_database_tuple(cursor, data)
+    file_name = data.output_path
+    if os.path.exists(file_name):
+        try:
+            with open(file_name) as f:
+                return Response(str(f.read()), mimetype='text/csv')
+        except Exception as e:
+            return "Error with file: " + Exception(e), 500
+    else:
+        return jsonify({"error": "Output file does not exist"}), 404
+
+
+@jobs_blueprint.route('/job', methods=["POST"])
+@jobs_blueprint.input(JobRequestSchema, location="json")
+def queue_job(json_data):
+    """
+    Adds a job to the queue
+    """
+    # Queue a job
+    if not (auth.authenticate_request(request)):
+        abort(403)
+        return
+    # Sometimes we need to get the form request (such as debugging from postman)
+    # Other times we output json
+    request_args = json_data
+
+    data_uri = "/api/temp/data"
+
+    if "data_uri" in request_args:
+        data_uri = request_args["data_uri"]
+
+    desc = ""
+    if "description" in request_args:
+        desc = request_args["description"]
+    conn = database.connect()
+    cursor = conn.cursor()
+
+    cursor.execute(
+        f"INSERT INTO hilsim_runs (user_id, branch, git_hash, submitted_time, output_path, run_status, description, data_uri) \
+                    VALUES (%s, %s, %s, now(), %s || currval ('hilsim_runs_run_id_seq'), %s, %s, %s) RETURNING run_id",
+        (request_args['username'],
+         request_args['branch'],
+         request_args['commit'],
+         JOB_OUTPUT_PREFIX,
+         0,
+         desc,
+         data_uri))
+    # TODO: Directory will be consructed later when the work actually starts
+    # https://github.com/orgs/ISSUIUC/projects/4/views/1?pane=issue&itemId=46405451
+
+    st = cursor.fetchall()
+    conn.commit()
+    conn.close()
+    cursor.close()
+    if len(st) > 0:
+        return jsonify({"status": "Job was created", "run_id": st[0][0]}), 201
+    else:
+        return jsonify({"error": "Error"}), 400
+
+@jobs_blueprint.route('/temp/data', methods=["GET"])
+def get_data():
+    """
+    Temporary data reader
+
+    TODO: delete this and replace with proper api stuff
+    https://github.com/orgs/ISSUIUC/projects/4/views/1?pane=issue&itemId=46405451
+    """
+    with open("./temp-data/flight_computer.csv") as f:
+        lines = f.read()
+        return lines

Central-Server/API/blueprints/perms.py

@@ -0,0 +1,22 @@
+from apiflask import APIBlueprint
+from flask import render_template, abort, jsonify, request, Response
+import requests
+
+
+perms_blueprint = APIBlueprint('perms', __name__)
+
+
+@perms_blueprint.route("/perms/<string:username>", methods=["GET"])
+def get_team_membership(username):
+    TOKEN = request.cookies.get('token')
+    # TOKEN = ''
+    url = f"https://api.github.com/orgs/ISSUIUC/teams/iss-kamaji-administrators/memberships/{username}"
+    x = requests.get(
+        url=url,
+        headers={
+            "Accept": "application/vnd.github+json",
+            'Authorization': f"Bearer {TOKEN}",
+            "Content-Type": "text/html; charset=utf-8",
+            "X-GitHub-Api-Version": "2022-11-28"})
+
+    return x.status_code == 200

Central-Server/API/database.sql

@@ -0,0 +1,23 @@
+/*
+Alright this is what I'm thinking:
+Run table
+run id, github user id of the person running, run, file path of the run output data, time of the run was submitted, run start time, runtime, and run end,run status
+run status enum:
+queued, running, cancelled, success, failed: crashed, failed: compile error, failed: timeout, failed: unknown
+Run config
+config id, other config stuff
+*/
+CREATE TABLE "hilsim_runs" (
+  "run_id" serial NOT NULL,
+  PRIMARY KEY ("run_id"),
+  "user_id" varchar(40) NOT NULL,
+  "branch" varchar(128) NOT NULL,
+  "git_hash" varchar(40) NOT NULL,
+  "output_path" varchar(128) NOT NULL,
+  "submitted_time" timestamp NOT NULL,
+  "run_start" timestamp NULL,
+  "run_end" timestamp NULL,
+  "run_status" smallint NOT NULL,
+  "description" varchar(512) NULL,
+  "data_uri" varchar(128) NULL
+);

Central-Server/API/internal/auth.py

@@ -0,0 +1,8 @@
+"""This file will eventually contain all the functions necessary to determine who a user is and what they can do."""
+from flask import Request
+
+
+def authenticate_request(xhr: Request) -> bool:
+    """Determine if the request has MEMBER-status permission (user is a member of ISSUIUC)"""
+    # https://github.com/orgs/ISSUIUC/projects/4/views/1?pane=issue&itemId=44397261
+    return True  # TODO: actual checks

Central-Server/API/internal/boards.py

@@ -0,0 +1,17 @@
+"""This file defines the schemas used by the Kamaji API when describing Datastreamer boards"""
+from apiflask import Schema
+from apiflask.fields import Integer, String, Boolean, List, Nested
+
+
+class BoardOuputSchema(Schema):
+    """Board state schema data container"""
+    id = Integer()  # The id of the board
+    is_ready = Boolean()  # True when in READY state (? TODO: check if this is true)
+    job_running = Boolean()  # True when actively running job
+    board_type = String()  # The type of board
+    running = Boolean()  # Is the thread currently running
+
+
+class BoardList(Schema):
+    """Class representing a collection of boards"""
+    boards = List(Nested(BoardOuputSchema))

Central-Server/API/internal/database.py

@@ -0,0 +1,78 @@
+"""This file exposes functions that abstract certain database functions"""
+import os
+import collections
+
+import psycopg2
+
+DATABASE_PORT = 5432  # Default postgres port
+
+
+def get_db_secret() -> str:
+    """Returns the database secret (password)"""
+    with open(os.getenv("DB_PASSWORD_FILE")) as f:
+        return (str(f.read()))
+
+
+db_secret = get_db_secret().strip()
+
+
+def get_db_name() -> str:
+    """Returns the name of the database used to store Kamaji data"""
+    val = os.getenv("DB_NAME")
+    if val is None:
+        return "db"
+    return val
+
+
+db_host = get_db_name()  # Exposed variable for getting database
+
+
+def connect():
+    """Returns a database connection after connecting with the DB credentials"""
+    global DATABASE_PORT
+    conn = psycopg2.connect(database="postgres",
+                            host=db_host,
+                            user="postgres",
+                            password=db_secret,
+                            port=DATABASE_PORT)
+    return conn
+
+
+def convert_database_tuple(cursor: psycopg2.extensions.cursor, data: tuple):
+    """
+    Normally, database output is in a struct, but we can fix that
+    @param data Tuple of data from the psycopg2 function cursor.fetchall()[0] or its equivalent
+    @returns struct of the record in a namedtuple
+    """
+    cols = [desc[0] for desc in cursor.description]
+    Record = collections.namedtuple("JobRecord", cols)
+    return Record(**dict(zip(cols, data)))
+
+
+def convert_database_list(
+        cursor: psycopg2.extensions.cursor,
+        data: list) -> list:
+    """
+    @param data List of data from the psycopg2 function cursor.fetchall()
+    @returns struct of the record in a namedtuple
+    """
+    cols = [desc[0] for desc in cursor.description]
+    Record = collections.namedtuple("JobRecord", cols)
+    record_list = []
+    for row in data:
+        record_list.append(Record(**dict(zip(cols, row))))
+    return record_list
+
+
+def generate_jobs_table():
+    """Generates database if it doesn't exist"""
+    conn = connect()
+    cursor = conn.cursor()
+    cursor.execute("SELECT 1 FROM pg_tables WHERE tablename='hilsim_runs'")
+    exists = cursor.fetchone()
+    if not exists:
+        with open("database.sql") as f:
+            cursor.execute(f.read())
+        conn.commit()
+        conn.close()
+        cursor.close()