Integration of Jupyter through docker-compose (#1088)

* Working Jupyterhub Integration * Linting * Handling logout * Updating docker-compose to keep jupyterhub version static * Incorporating feedback * Working Jupyterhub Integration * Linting * Handling logout * Updating docker-compose to keep jupyterhub version static * Incorporating feedback * Fix conflict * Fix config * Creating a traefik setup for prod docker-compose * Make it work for dev set up * Updating .env ecample * better file names * Updated prod webpack and a shell script for prod setup --------- Co-authored-by: Luigi Marini <[email protected]> Co-authored-by: Chen Wang <[email protected]>
clowder-framework · Jul 11, 2024 · f982515 · f982515
1 parent c5aa719
commit f982515
Show file tree

Hide file tree

Showing 21 changed files with 637 additions and 2 deletions.
diff --git a/.gitignore b/.gitignore
@@ -82,3 +82,4 @@ secrets.yaml
 # faker
 official.csv
 fact.png
+jupyterhub/.env-dev
diff --git a/README.md b/README.md
@@ -36,7 +36,9 @@ There is a few other documentation links available on the [website](https://clow
 ## Installation
 
 The easiest way of running Clowder v2 is checking out the [code](https://github.com/clowder-framework/clowder2)
-and running `docker compose up` in the main directory.
+and running `docker compose up` in the main directory. If you would like to run Clowder with JupyterHub,
+you can use our script `docker-prod.sh` to start the services. Run `./docker-prod.sh prod up` to start the services
+and `./docker-prod.sh prod down` to stop them.
 
 Helm charts are available for running Clowder v2 on Kubernetes. See the [helm](https://github.com/clowder-framework/clowder2/tree/main/deployments/kubernetes/charts) directory for more information.
 

diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml
@@ -67,6 +67,8 @@ services:
 
  postgres:
  image: postgres
+ networks:
+ - clowder2
  volumes:
  - postgres_data:/var/lib/postgresql/data
  environment:
@@ -79,6 +81,8 @@ services:
  volumes:
  - ./scripts/keycloak/clowder-realm-dev.json:/opt/keycloak/data/import/realm.json:ro
  - ./scripts/keycloak/clowder-theme/:/opt/keycloak/themes/clowder-theme/:ro
+ networks:
+ - clowder2
  command:
  - start-dev
  - --http-relative-path /keycloak
@@ -172,6 +176,7 @@ services:
 
 networks:
  clowder2:
+ name: clowder2
 
 ## By default this config uses default local driver,
 ## For custom volumes replace with volume driver configuration.

diff --git a/docker-compose.jupyter-dev.yml b/docker-compose.jupyter-dev.yml
@@ -0,0 +1,30 @@
+version: '3'
+services:
+ jupyterhub:
+ build:
+ context: jupyterhub
+ dockerfile: Dockerfile.jupyterhub
+ args:
+ JUPYTERHUB_VERSION: 4
+ restart: always
+ networks:
+ - clowder2
+ volumes:
+ # The JupyterHub configuration file
+ - ./jupyterhub/jupyterhub_dev_config.py:/srv/jupyterhub/jupyterhub_config.py:ro
+ # Bind Docker socket on the hostso we can connect to the daemon from
+ # within the container
+ - /var/run/docker.sock:/var/run/docker.sock:rw
+ # Bind Docker volume on host for JupyterHub database and cookie secrets
+ - jupyterhub-data:/data
+ ports:
+ - "8765:8000"
+ env_file:
+ - jupyterhub/.env-dev
+ command: jupyterhub -f /srv/jupyterhub/jupyterhub_config.py
+
+ depends_on:
+ - keycloak
+
+volumes:
+ jupyterhub-data:
diff --git a/docker-compose.jupyter.yml b/docker-compose.jupyter.yml
@@ -0,0 +1,33 @@
+version: '3'
+services:
+ jupyterhub:
+ build:
+ context: jupyterhub
+ dockerfile: Dockerfile.jupyterhub
+ args:
+ JUPYTERHUB_VERSION: 4
+ restart: always
+ networks:
+ - clowder2
+ volumes:
+ # The JupyterHub configuration file
+ - ./jupyterhub/jupyterhub_config.py:/srv/jupyterhub/jupyterhub_config.py:ro
+ # Bind Docker socket on the hostso we can connect to the daemon from
+ # within the container
+ - /var/run/docker.sock:/var/run/docker.sock:rw
+ # Bind Docker volume on host for JupyterHub database and cookie secrets
+ - jupyterhub_data:/data
+ env_file:
+ - jupyterhub/.env
+ labels:
+ - "traefik.enable=true"
+ - "traefik.http.routers.jupyterhub.rule=PathPrefix(`/jupyterhub`)"
+ - "traefik.http.services.jupyterhub.loadbalancer.server.port=8000"
+
+ command: jupyterhub -f /srv/jupyterhub/jupyterhub_config.py
+
+ depends_on:
+ - keycloak
+
+volumes:
+ jupyterhub_data:
diff --git a/docker-dev.sh b/docker-dev.sh
@@ -7,3 +7,12 @@ if [ "$1" = "down" ]
 then
  docker-compose -f docker-compose.dev.yml -p clowder2-dev down
 fi
+if [ "$1" = "jupyter" ] && [ "$2" = "up" ]
+then
+ docker-compose -f docker-compose.dev.yml -f docker-compose.jupyter-dev.yml -p clowder2-dev up -d --build
+fi
+
+if [ "$1" = "jupyter" ] && [ "$2" = "down" ]
+then
+ docker-compose -f docker-compose.dev.yml -f docker-compose.jupyter-dev.yml -p clowder2-dev down
+fi
diff --git a/docker-prod.sh b/docker-prod.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env sh
+if [ "$1" = "prod" ] && [ "$2" = "up" ]
+then
+ docker-compose -f docker-compose.yml -f docker-compose.jupyter.yml up -d
+fi
+
+if [ "$1" = "prod" ] && [ "$2" = "down" ]
+then
+ docker-compose -f docker-compose.yml -f docker-compose.jupyter.yml down
+fi
diff --git a/docs/docs/devs/getstarted.md b/docs/docs/devs/getstarted.md
@@ -47,6 +47,8 @@ section below).
 - Running `docker-compose logs -f` displays the live logs for all containers. To view the logs of individual containers,
  provide the container name. For example, for viewing the backend module logs, run `docker-compose logs -f backend`.
 - Running `./docker-dev.sh down` brings down the required services.
+- If you want to run the jupyterhub, you can run `./docker-dev.sh jupyter up`. The jupyterhub will be available at
+ `http://localhost:8765`. You can bring it down using `./docker-dev.sh jupyter down`.
 
 **Note:** `./docker-dev.sh` sets the project name flag to `-p clowder2-dev`. This is so that the dev containers
 don't get mixed with the production containers if the user is running both on the same machine using `docker-compose.yml`.

diff --git a/frontend/package.json b/frontend/package.json
@@ -12,7 +12,7 @@
  "start-message": "babel-node tools/startMessage.js",
  "prestart": "npm-run-all --parallel start-message",
  "start": "npm-run-all --parallel open:src",
- "start:dev": "export CLOWDER_REMOTE_HOSTNAME=http://localhost:8000 && npm run start",
+ "start:dev": "export CLOWDER_REMOTE_HOSTNAME=http://localhost:8000 && export JUPYTERHUB_URL=http://localhost:8765 && npm run start",
  "open:src": "babel-node tools/srcServer.js",
  "open:dist": "babel-node tools/distServer.js",
  "lint:watch": "npm run lint --watch",

diff --git a/frontend/src/app.config.ts b/frontend/src/app.config.ts
@@ -9,6 +9,7 @@ interface Config {
  hostname: string;
  apikey: string;
  GHIssueBaseURL: string;
+ jupyterHubURL: string;
  KeycloakBaseURL: string;
  KeycloakLogin: string;
  KeycloakLogout: string;
@@ -66,6 +67,10 @@ config["KeycloakRegister"] = `${config.KeycloakBaseURL}/register`;
 config["searchEndpoint"] = `${hostname}/api/v2/elasticsearch`;
 config["publicSearchEndpoint"] = `${hostname}/api/v2/public_elasticsearch`;
 
+// jupterhub
+const localJupyterhubURL: string = `${config.hostname}/jupyterhub`;
+config["jupyterHubURL"] = process.env.JUPYTERHUB_URL || localJupyterhubURL;
+
 // refresh token time interval
 config["refreshTokenInterval"] = 1000 * 60; // 1 minute
 // updated extractor logs

diff --git a/frontend/src/components/Layout.tsx b/frontend/src/components/Layout.tsx
@@ -22,6 +22,7 @@ import { RootState } from "../types/data";
 import { AddBox, Explore } from "@material-ui/icons";
 import HistoryIcon from "@mui/icons-material/History";
 import GroupIcon from "@mui/icons-material/Group";
+import MenuBookIcon from "@mui/icons-material/MenuBook";
 import Gravatar from "react-gravatar";
 import PersonIcon from "@mui/icons-material/Person";
 import InfoOutlinedIcon from "@mui/icons-material/InfoOutlined";
@@ -40,6 +41,8 @@ import AdminPanelSettingsIcon from "@mui/icons-material/AdminPanelSettings";
 import { Footer } from "./navigation/Footer";
 import BuildIcon from "@mui/icons-material/Build";
 
+import config from "../app.config";
+
 const drawerWidth = 240;
 
 const Main = styled("main", { shouldForwardProp: (prop) => prop !== "open" })<{
@@ -422,6 +425,22 @@ export default function PersistentDrawerLeft(props) {
  </ListItemButton>
  </ListItem>
  </List>
+ {/*TODO: Need to make link dynamic */}
+ <List>
+ <ListItem key={"jupyter"} disablePadding>
+ <ListItemButton
+ component={RouterLink}
+ to={config.jupyterHubURL}
+ target="_blank"
+ rel="noopener noreferrer"
+ >
+ <ListItemIcon>
+ <MenuBookIcon />
+ </ListItemIcon>
+ <ListItemText primary={"Jupyter Notebook"} />
+ </ListItemButton>
+ </ListItem>
+ </List>
  <List>
  <ListItem key={"feeds"} disablePadding>
  <ListItemButton component={RouterLink} to="/feeds">

diff --git a/frontend/webpack.config.dev.js b/frontend/webpack.config.dev.js
@@ -8,6 +8,9 @@ import ESLintPlugin from "eslint-webpack-plugin";
 console.log(
  `the current CLOWDER_REMOTE_HOSTNAME environment variable is ${process.env.CLOWDER_REMOTE_HOSTNAME}`
 );
+console.log(
+ `the JupyterHub URL is set to ${process.env.JUPYTERHUB_URL}`
+)
 
 export default {
  mode: "development",
@@ -40,6 +43,7 @@ export default {
  CLOWDER_REMOTE_HOSTNAME: JSON.stringify(
  process.env.CLOWDER_REMOTE_HOSTNAME
  ),
+ JUPYTERHUB_URL: JSON.stringify(process.env.JUPYTERHUB_URL),
  APIKEY: JSON.stringify(process.env.APIKEY),
  KeycloakBaseURL: JSON.stringify(process.env.KeycloakBaseURL),
  },

diff --git a/frontend/webpack.config.prod.js b/frontend/webpack.config.prod.js
@@ -11,6 +11,9 @@ import TerserPlugin from "terser-webpack-plugin";
 console.log(
  `the current CLOWDER_REMOTE_HOSTNAME environment variable is ${process.env.CLOWDER_REMOTE_HOSTNAME}`
 );
+console.log(
+ `the JupyterHub URL is set to ${process.env.JUPYTERHUB_URL}`
+)
 
 export default {
  mode: "production",
@@ -47,6 +50,7 @@ export default {
  CLOWDER_REMOTE_HOSTNAME: JSON.stringify(
  process.env.CLOWDER_REMOTE_HOSTNAME
  ),
+ JUPYTERHUB_URL: JSON.stringify(process.env.JUPYTERHUB_URL),
  APIKEY: JSON.stringify(process.env.APIKEY),
  KeycloakBaseURL: JSON.stringify(process.env.KeycloakBaseURL),
  },

diff --git a/jupyterhub/.env-example b/jupyterhub/.env-example
@@ -0,0 +1,14 @@
+# Example configuration file for Clowder JupyterHub
+KEYCLOAK_HOSTNAME="keycloak:8080/keycloak"
+# Development mode use the following line instead
+#KEYCLOAK_HOSTNAME="keycloak:8080/keycloak"
+KEYCLOAK_AUDIENCE="clowder"
+KEYCLOAK_REALM="clowder"
+JUPYTERHUB_ADMIN="admin"
+#Change network name to the one created by docker-compose
+DOCKER_NETWORK_NAME="clowder2_clowder2"
+DOCKER_NOTEBOOK_IMAGE="quay.io/jupyter/base-notebook:latest"
+DOCKER_NOTEBOOK_DIR="/home/jovyan/work"
+JUPYTERHUB_CRYPT_KEY=""
+CLOWDER_URL="localhost"
+PROD_DEPLOYMENT="false"
diff --git a/jupyterhub/Dockerfile.jupyterhub b/jupyterhub/Dockerfile.jupyterhub
@@ -0,0 +1,14 @@
+ARG JUPYTERHUB_VERSION
+FROM quay.io/jupyterhub/jupyterhub:$JUPYTERHUB_VERSION
+
+# Install dockerspawner,
+# hadolint ignore=DL3013
+RUN python3 -m pip install --no-cache-dir \
+ dockerspawner
+
+# Install custom authenticator
+WORKDIR /tmp/authenticator/
+COPY authenticator /tmp/authenticator/
+RUN pip3 install /tmp/authenticator
+
+CMD ["jupyterhub", "-f", "/srv/jupyterhub/jupyterhub_config.py"]
diff --git a/jupyterhub/authenticator/customauthenticator/__init__.py b/jupyterhub/authenticator/customauthenticator/__init__.py