Added initial subset of config for hub, spawner, and singularity note…

…books.

Not much in the way of documentation yet, mostly just bare config lifted out of our deployment for jupyterhub#5. Still a WIP.

optionsspawner-slurm-singularity-rmaccsummit/README.md

@@ -0,0 +1,2 @@
+Jupyter-Summit deployment at University of Colorado Boulder Research Computing
+---

optionsspawner-slurm-singularity-rmaccsummit/config/form_config.py

@@ -0,0 +1,89 @@
+from optionsspawner.forms import (
+    FormField,
+    TextInputField,
+    NumericalInputField,
+    CheckboxInputField,
+    SelectField,
+)
+
+
+partition_select = SelectField('req_partition',
+    label='Select a partition',
+    attr_required=True,
+    choices=[
+        ('shas', "Summit - Haswell"),
+        ('sknl', "Summit - Knight's Landing"),
+        ('blanca-csdms', "Blanca - CSDMS"),
+        ('blanca-sol', "Blanca - Sol"),
+    ],
+    default='shas'
+)
+
+qos_select = SelectField('req_qos',
+    label='Select a QoS',
+    attr_required=True,
+    choices=[
+        ('jupyterhub', "Summit - All Partitions"),
+        ('blanca-csdms', "Blanca - CSDMS"),
+        ('blanca-sol', "Blanca - Sol"),
+    ],
+    default='jupyterhub'
+)
+
+account_input = TextInputField('req_account',
+    label='Specify an account to charge (Required for Blanca users)'
+)
+
+cluster_select = SelectField('req_cluster',
+    label='Select a cluster',
+    attr_required=True,
+    choices=[
+        ('summit', "Summit"),
+        ('blanca', "Blanca"),
+    ],
+)
+
+runtime_input = TextInputField('req_runtime',
+    label='Specify runtime (HH:MM:SS)',
+    attr_required=True,
+    attr_value='02:00:00',
+    attr_pattern="([01]?[0-9]{1}|2[0-3]{1}):[0-5]{1}[0-9]{1}:[0-5]{1}[0-9]{1}"
+)
+
+nodes_input = NumericalInputField('req_nodes',
+    label='Specify node count',
+    attr_required=True,
+    attr_value=1,
+    attr_min=1,
+    attr_max=4
+)
+
+ntasks_input = NumericalInputField('req_ntasks',
+    label='Specify tasks per node',
+    attr_required=True,
+    attr_value=1,
+    attr_min=1,
+    attr_max=24
+)
+
+image_select = SelectField('req_image_path',
+    label='Select a notebook image',
+    attr_required=True,
+    choices=[
+        ('/curc/tools/images/jupyter-notebook-base/jupyter-notebook-baselatest.simg', "Python3"),
+        ('/curc/tools/images/jupyter-notebook-base/jupyter-notebook-pysparklatest.simg', "PySpark"),
+        ('/curc/tools/images/jupyter-notebook-base/jupyter-notebook-rlatest.simg', "R"),
+    ],
+    default='/curc/tools/images/jupyter-notebook-base/jupyter-notebook-baselatest.simg'
+)
+
+form_fields = [
+    cluster_select,
+    partition_select,
+    qos_select,
+    account_input,
+    image_select,
+    runtime_input,
+    nodes_input,
+    ntasks_input,
+]

optionsspawner-slurm-singularity-rmaccsummit/config/jupyterhub_config.py

@@ -0,0 +1,54 @@
+import os
+import importlib.machinery
+
+slurm_config = importlib.machinery.SourceFileLoader('slurm_config','/etc/jupyterhub/config/slurm_config.py').load_module()
+form_config = importlib.machinery.SourceFileLoader('form_config','/etc/jupyterhub/config/form_config.py').load_module()
+
+
+
+# Set the log level by value or name.
+c.JupyterHub.log_level = 'DEBUG'
+c.JupyterHub.extra_log_file = '/etc/jupyterhub/debug-log.log'
+
+# Allow servers to persist between restarts of the hub itself
+c.JupyterHub.cleanup_servers = False
+
+# SPAWNER CONFIGURATION
+
+# Increase spawner timeout to be tolerant of longer queue wait times.
+c.Spawner.http_timeout = 300
+c.Spawner.start_timeout = 300
+
+# https://github.com/jupyterhub/jupyterhub/issues/929
+c.Spawner.notebook_dir = '/'
+c.Spawner.default_url = '/tree/home/{username}'
+
+# OptionsSpawner: Attach options forms to any JupyterHub spawner using only configuration.
+# https://github.com/ResearchComputing/jupyterhub-options-spawner
+c.JupyterHub.spawner_class = 'optionsspawner.OptionsFormSpawner'
+c.OptionsFormTestSpawner.debug = True
+# The OptionsSpawner wraps SlurmSpawner from https://github.com/jupyterhub/batchspawner
+c.OptionsFormSpawner.child_class = 'batchspawner.SlurmSpawner'
+c.OptionsFormSpawner.child_config = slurm_config.spawner_config
+c.OptionsFormSpawner.form_fields = form_config.form_fields
+
+
+c.JupyterHub.ip = '0.0.0.0'
+c.JupyterHub.port = 8000
+
+jupyterhub_hostname = os.environ.get('HOSTNAME')
+c.JupyterHub.hub_ip = jupyterhub_hostname
+
+c.JupyterHub.cookie_secret_file = '/etc/jupyterhub/jupyterhub_cookie_secret'
+
+c.JupyterHub.db_url = '/etc/jupyterhub/jupyterhub.sqlite'
+
+c.JupyterHub.ssl_cert = '/etc/jupyterhub/certs/jupyter.crt'
+c.JupyterHub.ssl_key = '/etc/jupyterhub/certs/jupyter.key'
+
+admins_env = os.environ.get('JUPYTERHUB_ADMINS', '')
+admins = tuple(admins_env.split()) if admins_env else ()
+c.Authenticator.admin_users = admins
+c.JupyterHub.admin_access = True
+
+#c.JupyterHub.template_paths = ["/opt/jupyterhub/jupyterhub-branded-pages/templates"]

optionsspawner-slurm-singularity-rmaccsummit/config/slurm_config.py

@@ -0,0 +1,33 @@
+summit_script = """#!/bin/bash
+#SBATCH --partition={partition}
+#SBATCH --qos={qos}
+#SBATCH --account={account}
+#SBATCH --time={runtime}
+#SBATCH --nodes={nodes}
+#SBATCH --ntasks-per-node={ntasks}
+#SBATCH --output={homedir}/.jupyterhub-slurmspawner.log
+#SBATCH --open-mode=append
+#SBATCH --job-name=spawner-jupyterhub
+#SBATCH --workdir={homedir}
+#SBATCH --export={keepvars}
+#SBATCH --uid={username}
+
+ml singularity/2.4.2
+
+# jupyter-singleuser anticipates that environment will be dropped during sudo, however
+# it is retained by batchspawner. The XDG_RUNTIME_DIR variable must be unset to force a
+# fallback, otherwise a permissions error occurs when starting the notebook.
+# https://github.com/jupyter/notebook/issues/1318
+
+export SINGULARITYENV_JUPYTERHUB_API_TOKEN=$JUPYTERHUB_API_TOKEN
+export SINGULARITYENV_XDG_RUNTIME_DIR=$HOME/.singularity-jupyter-run
+export SINGULARITYENV_CONTAINER_PATH={image_path}
+singularity run --bind /var/lib/sss/pipes --bind /home/slurm --bind /var/run/munge --bind /etc/slurm --bind /curc/slurm --bind /etc/pam.d $SINGULARITYENV_CONTAINER_PATH {cmd}
+"""
+
+spawner_config = {
+    'batch_script': summit_script,
+    'batch_submit_cmd': """sudo -E -u {username} SLURM_CONF=/curc/slurm/{cluster}/etc/slurm.conf sbatch""",
+    'batch_query_cmd': """sudo -E -u {username} SLURM_CONF=/curc/slurm/{cluster}/etc/slurm.conf squeue -h -j {job_id} -o "%T %B" """,
+    'batch_cancel_cmd': """sudo -E -u {username} SLURM_CONF=/curc/slurm/{cluster}/etc/slurm.conf scancel {job_id}""",
+}

optionsspawner-slurm-singularity-rmaccsummit/singularity-notebook-ipyparallel/README.md

@@ -0,0 +1,9 @@
+# jupyter-notebook-base
+Singularity base image for Jupyter Notebook servers running on RC resources. Can be started either via JupyterHub or directly by the end-user in a tunneling setup.
+
+## Usage
+```
+singularity image.create jupyter-notebook-base.img
+singularity build jupyter-notebook-base.img Singularity
+singularity shell --bind /var/lib/sss/pipes --bind /home/slurm --bind /var/run/munge --bind /etc/slurm --bind /curc/slurm jupyter-notebook-base.img
+```

optionsspawner-slurm-singularity-rmaccsummit/singularity-notebook-ipyparallel/Singularity

@@ -0,0 +1,59 @@
+Bootstrap: docker
+From: centos:7
+
+%labels
+  MAINTAINER [email protected]
+
+%environment
+  export JUPYTER_DATA_DIR=$HOME/.singularity-jupyter
+
+%setup
+  mkdir -p /usr/local/share/profile_example-shas
+
+%files
+  profile_example-shas/ /usr/local/share/profile_example-shas/
+
+%post
+  # Install dependencies for Slurm, SSSD, and PAM
+  useradd -u 515 -m slurm
+  useradd -u 992 -m munge
+  yum -y update
+  yum -y install epel-release
+  yum -y groupinstall 'Development Tools'
+  yum -y install sssd curl wget strace iproute munge munge-devel pam-devel openssl openssl-devel readline-devel perl-devel
+  cd ~ && wget https://download.schedmd.com/slurm/slurm-17.02.9.tar.bz2
+  rpmbuild -ta slurm-17.02.9.tar.bz2
+  cd ~/rpmbuild/RPMS/x86_64
+  rpm -ivh slurm-pam_slurm-17.02.9-1.el7.centos.x86_64.rpm slurm-plugins-17.02.9-1.el7.centos.x86_64.rpm slurm-munge-17.02.9-1.el7.centos.x86_64.rpm slurm-perlapi-17.02.9-1.el7.centos.x86_64.rpm slurm-17.02.9-1.el7.centos.x86_64.rpm slurm-devel-17.02.9-1.el7.centos.x86_64.rpm
+
+  # Install OpenMPI
+  # Omnipath and OpenMPI user libraries for Summit
+  yum install -y libhfi1 libpsm2 libpsm2-devel libpsm2-compat
+  yum install -y perftest qperf
+  yum install -y libibverbs libibverbs-devel rdma
+  yum install -y numactl-libs numactl-devel
+  yum install -y pciutils
+  yum install -y which
+  wget https://download.open-mpi.org/release/open-mpi/v2.0/openmpi-2.0.1.tar.gz
+  tar -xf openmpi-2.0.1.tar.gz
+  cd openmpi-2.0.1/
+  ./configure \
+  --with-verbs \
+  --with-psm2 \
+  --enable-mpi-thread-multiple
+  make -j2
+  make install
+
+  # Install Jupyter*
+  yum -y install python34-devel python34-pip
+  pip3 install --upgrade pip
+  pip3 install jupyterhub==0.7.2
+  pip3 install --upgrade notebook
+  pip3 install ipyparallel==6.2
+  pip3 install matplotlib numpy scipy pandas
+
+%runscript
+  # Copy example ipyparallel profile to home directory
+  mkdir -p $HOME/.ipython
+  cp -rf /usr/local/share/profile_example-shas $HOME/.ipython/profile_example-shas
+  exec "$@"

optionsspawner-slurm-singularity-rmaccsummit/singularity-notebook-ipyparallel/profile_example-shas/ipcluster_config.py

@@ -0,0 +1,38 @@
+# Configuration file for ipcluster.
+
+c.IPClusterEngines.engine_launcher_class = 'ipyparallel.apps.launcher.SlurmEngineSetLauncher'
+c.SlurmLauncher.qos = 'normal'
+c.SlurmLauncher.timelimit = '1:00:00'
+
+c.SlurmEngineSetLauncher.batch_template = """#!/bin/bash
+
+#SBATCH --partition shas
+#SBATCH --qos {qos}
+#SBATCH --job-name ipengine
+#SBATCH --ntasks {n}
+# This will run a single ipengine per CPU
+#SBATCH --cpus-per-task 1
+# Use ntasks-per-node=1 to run one ipengine per node
+#SBATCH --time {timelimit}
+#SBATCH --output {profile_dir}/log/slurm.out
+
+ml gcc singularity/2.4.2
+ml openmpi/2.0.1
+
+# jupyter-singleuser anticipates that environment will be dropped during sudo, however
+# it is retained by batchspawner. The XDG_RUNTIME_DIR variable must be unset to force a
+# fallback, otherwise a permissions error occurs when starting the notebook.
+# https://github.com/jupyter/notebook/issues/1318
+
+export SINGULARITYENV_JUPYTERHUB_API_TOKEN=$JUPYTERHUB_API_TOKEN
+export SINGULARITYENV_XDG_RUNTIME_DIR=$HOME/.singularity-jupyter-run
+export SINGULARITYENV_CONTAINER_PATH=$CONTAINER_PATH
+mpirun singularity run \
+  --bind /var/lib/sss/pipes \
+  --bind /home/slurm \
+  --bind /var/run/munge \
+  --bind /etc/slurm \
+  --bind /etc/pam.d \
+  $CONTAINER_PATH \
+  ipengine --profile-dir="{profile_dir}" --cluster-id="{cluster_id}"
+"""

optionsspawner-slurm-singularity-rmaccsummit/singularity-notebook-ipyparallel/profile_example-shas/ipcontroller_config.py

@@ -0,0 +1,3 @@
+# Configuration file for ipcontroller.
+
+c.HubFactory.ip = '*'

optionsspawner-slurm-singularity-rmaccsummit/singularity-notebook-ipyparallel/profile_example-shas/startup/README

@@ -0,0 +1,11 @@
+This is the IPython startup directory
+
+.py and .ipy files in this directory will be run *prior* to any code or files specified
+via the exec_lines or exec_files configurables whenever you load this profile.
+
+Files will be run in lexicographical order, so you can control the execution order of files
+with a prefix, e.g.::
+
+    00-first.py
+    50-middle.py
+    99-last.ipy