From b3217dc9fc08d52a04da514d25183eca49ea61eb Mon Sep 17 00:00:00 2001
From: "Michael J. Stealey" <michael.j.stealey@gmail.com>
Date: Fri, 18 May 2018 22:14:18 -0400
Subject: [PATCH] #20 nfs mounts

Initial version of NFS mounted volumes
---
 base/Dockerfile                 |   3 +-
 controller/Dockerfile           |   5 +-
 controller/docker-entrypoint.sh |  41 +++++++++++
 database/Dockerfile             |   5 +-
 database/docker-entrypoint.sh   |  41 +++++++++++
 docker-compose-lmod-nfs.yml     | 117 ++++++++++++++++++++++++++++++++
 nfs-server/Dockerfile           |  17 +++++
 nfs-server/docker-entrypoint.sh |  93 +++++++++++++++++++++++++
 worker/Dockerfile               |   4 ++
 worker/docker-entrypoint.sh     |  41 +++++++++++
 10 files changed, 364 insertions(+), 3 deletions(-)
 create mode 100644 docker-compose-lmod-nfs.yml
 create mode 100644 nfs-server/Dockerfile
 create mode 100755 nfs-server/docker-entrypoint.sh

diff --git a/base/Dockerfile b/base/Dockerfile
index 9f4693c..3e44a06 100644
--- a/base/Dockerfile
+++ b/base/Dockerfile
@@ -27,7 +27,8 @@ RUN yum -y install \
   munge-libs \
   munge-devel \
   openssh-server \
-  openssh-clients
+  openssh-clients \
+  nfs-utils
 
 # install slurm 17.11.5
 COPY rpms /packages
diff --git a/controller/Dockerfile b/controller/Dockerfile
index e31443b..76acc8e 100644
--- a/controller/Dockerfile
+++ b/controller/Dockerfile
@@ -34,7 +34,10 @@ ENV USE_SLURMDBD=true \
   SLURMD_PORT=6818 \
   ACCOUNTING_STORAGE_HOST=database \
   ACCOUNTING_STORAGE_PORT=6819 \
-  PARTITION_NAME=docker
+  PARTITION_NAME=docker \
+  NFS_SERVER=nfs-server \
+  NFS_SERVER_DIRS=/nfs/secret:/nfs/home:/nfs/modules:/nfs/modulefiles \
+  NFS_CLIENT_DIRS=/secret:/home:/opt/apps/Linux:/opt/apps/modulefiles/Linux
 
 # clean up
 RUN rm -f /packages/slurm-*.rpm /packages/openmpi-*.rpm \
diff --git a/controller/docker-entrypoint.sh b/controller/docker-entrypoint.sh
index 0bcf241..07eb2d8 100755
--- a/controller/docker-entrypoint.sh
+++ b/controller/docker-entrypoint.sh
@@ -1,6 +1,39 @@
 #!/usr/bin/env bash
 set -e
 
+_init_fstab() {
+  if [[ ! -f /etc/fstab ]]; then
+    cat > /etc/fstab << EOF
+### <server>:</remote/export> </local/directory> <nfs-type> <options> 0 0
+EOF
+  fi
+}
+
+_export_nfs_mounts() {
+  IFS=':' read -r -a MNT_SERVER_ARRAY <<< "$NFS_SERVER_DIRS"
+  IFS=':' read -r -a MNT_CLIENT_ARRAY <<< "$NFS_CLIENT_DIRS"
+  for i in "${!MNT_CLIENT_ARRAY[@]}"; do
+    if [[ ! -d ${MNT_CLIENT_ARRAY[$i]} ]]; then
+      mkdir -p ${MNT_CLIENT_ARRAY[$i]}
+    fi
+    cat >> /etc/fstab <<EOF
+${NFS_SERVER}:${MNT_SERVER_ARRAY[$i]} ${MNT_CLIENT_ARRAY[$i]} nfs rw,hard,intr 0 0
+EOF
+  done
+  cat /etc/fstab
+}
+
+_start_nfs_services() {
+  rpcbind
+  rpc.nfsd
+  echo "connecting to ${NFS_SERVER}"
+  sleep 2
+  until [ $(ping ${NFS_SERVER} -c 3 2>&1 >/dev/null)$? ]; do
+    echo -n "."
+    sleep 2
+  done
+}
+
 # start sshd server
 _sshd_host() {
   if [ ! -d /var/run/sshd ]; then
@@ -193,6 +226,14 @@ _slurmctld() {
 }
 
 ### main ###
+_start_nfs_services
+_init_fstab
+_export_nfs_mounts
+mount -a
+
+rpcinfo -p $NFS_SERVER
+showmount -e $NFS_SERVER
+
 _sshd_host
 _ssh_worker
 _munge_start
diff --git a/database/Dockerfile b/database/Dockerfile
index 8763ed5..315bc98 100644
--- a/database/Dockerfile
+++ b/database/Dockerfile
@@ -7,7 +7,10 @@ ENV DBD_ADDR=database \
   STORAGE_HOST=database.local.dev \
   STORAGE_PORT=3306 \
   STORAGE_PASS=password \
-  STORAGE_USER=slurm
+  STORAGE_USER=slurm \
+  NFS_SERVER=nfs-server \
+  NFS_SERVER_DIRS=/nfs/secret:/nfs/home:/nfs/modules:/nfs/modulefiles \
+  NFS_CLIENT_DIRS=/secret:/home:/opt/apps/Linux:/opt/apps/modulefiles/Linux
 
 # clean up
 RUN rm -f /packages/slurm-*.rpm /packages/openmpi-*.rpm \
diff --git a/database/docker-entrypoint.sh b/database/docker-entrypoint.sh
index 81cd307..e81acc2 100755
--- a/database/docker-entrypoint.sh
+++ b/database/docker-entrypoint.sh
@@ -3,6 +3,39 @@ set -e
 
 SLURM_ACCT_DB_SQL=/slurm_acct_db.sql
 
+_init_fstab() {
+  if [[ ! -f /etc/fstab ]]; then
+    cat > /etc/fstab << EOF
+### <server>:</remote/export> </local/directory> <nfs-type> <options> 0 0
+EOF
+  fi
+}
+
+_export_nfs_mounts() {
+  IFS=':' read -r -a MNT_SERVER_ARRAY <<< "$NFS_SERVER_DIRS"
+  IFS=':' read -r -a MNT_CLIENT_ARRAY <<< "$NFS_CLIENT_DIRS"
+  for i in "${!MNT_CLIENT_ARRAY[@]}"; do
+    if [[ ! -d ${MNT_CLIENT_ARRAY[$i]} ]]; then
+      mkdir -p ${MNT_CLIENT_ARRAY[$i]}
+    fi
+    cat >> /etc/fstab <<EOF
+${NFS_SERVER}:${MNT_SERVER_ARRAY[$i]} ${MNT_CLIENT_ARRAY[$i]} nfs rw,hard,intr 0 0
+EOF
+  done
+  cat /etc/fstab
+}
+
+_start_nfs_services() {
+  rpcbind
+  rpc.nfsd
+  sleep 2
+  echo "connecting to ${NFS_SERVER}"
+  until [ $(ping ${NFS_SERVER} -c 3 2>&1 >/dev/null)$? ]; do
+    echo -n "."
+    sleep 2
+  done
+}
+
 # start sshd server
 _sshd_host() {
   if [ ! -d /var/run/sshd ]; then
@@ -130,6 +163,14 @@ _slurmdbd() {
 }
 
 ### main ###
+_start_nfs_services
+_init_fstab
+_export_nfs_mounts
+mount -a
+
+rpcinfo -p $NFS_SERVER
+showmount -e $NFS_SERVER
+
 _sshd_host
 _mariadb_start
 _munge_start_using_key
diff --git a/docker-compose-lmod-nfs.yml b/docker-compose-lmod-nfs.yml
new file mode 100644
index 0000000..038d44d
--- /dev/null
+++ b/docker-compose-lmod-nfs.yml
@@ -0,0 +1,117 @@
+version: '3.1'
+
+services:
+  nfs-server:
+    build:
+      context: ./nfs-server
+      dockerfile: Dockerfile
+    image: nfs-server
+    container_name: nfs-server
+    privileged: true
+    hostname: nfs-server.local.dev
+    networks:
+      - slurm
+    environment:
+      RPCNFSDCOUNT: 16
+      NFS_SERVER_DIRS: '/nfs/secret:/nfs/home:/nfs/modules:/nfs/modulefiles'
+    volumes:
+      - nfs-vol:/nfs
+
+  controller:
+    build:
+      context: ./controller
+      dockerfile: Dockerfile
+    image: scidas/slurm.controller:17.11.5
+    container_name: controller
+    privileged: true
+    restart: always
+    hostname: controller.local.dev
+    networks:
+      - slurm
+    environment:
+      USE_SLURMDBD: 'true'
+      CLUSTER_NAME: snowflake
+      CONTROL_MACHINE: controller
+      SLURMCTLD_PORT: 6817
+      SLURMD_PORT: 6818
+      ACCOUNTING_STORAGE_HOST: database
+      ACCOUNTING_STORAGE_PORT: 6819
+      COMPUTE_NODES: worker01 worker02
+      PARTITION_NAME: docker
+      NFS_SERVER: nfs-server
+      NFS_SERVER_DIRS: '/nfs/secret:/nfs/home:/nfs/modules:/nfs/modulefiles'
+      NFS_CLIENT_DIRS: '/.secret:/home:/opt/apps/Linux:/opt/apps/modulefiles/Linux'
+
+  database:
+    build:
+      context: ./database
+      dockerfile: Dockerfile
+    image: scidas/slurm.database:17.11.5
+    depends_on:
+      - controller
+    container_name: database
+    privileged: true
+    restart: always
+    hostname: database.local.dev
+    networks:
+      - slurm
+    environment:
+      DBD_ADDR: database
+      DBD_HOST: database
+      DBD_PORT: 6819
+      STORAGE_HOST: database.local.dev
+      STORAGE_PORT: 3306
+      STORAGE_PASS: password
+      STORAGE_USER: slurm
+      NFS_SERVER: nfs-server
+      NFS_SERVER_DIRS: '/nfs/secret:/nfs/home:/nfs/modules:/nfs/modulefiles'
+      NFS_CLIENT_DIRS: '/.secret:/home:/opt/apps/Linux:/opt/apps/modulefiles/Linux'
+
+  worker01:
+    build:
+      context: ./worker
+      dockerfile: Dockerfile
+    image: scidas/slurm.worker:17.11.5
+    depends_on:
+      - controller
+    container_name: worker01
+    privileged: true
+    restart: always
+    hostname: worker01.local.dev
+    networks:
+      - slurm
+    environment:
+      CONTROL_MACHINE: controller
+      ACCOUNTING_STORAGE_HOST: database
+      COMPUTE_NODES: worker01 worker02
+      NFS_SERVER: nfs-server
+      NFS_SERVER_DIRS: '/nfs/secret:/nfs/home:/nfs/modules:/nfs/modulefiles'
+      NFS_CLIENT_DIRS: '/.secret:/home:/opt/apps/Linux:/opt/apps/modulefiles/Linux'
+
+  worker02:
+    build:
+      context: ./worker
+      dockerfile: Dockerfile
+    image: scidas/slurm.worker:17.11.5
+    depends_on:
+      - controller
+    container_name: worker02
+    privileged: true
+    restart: always
+    hostname: worker02.local.dev
+    networks:
+      - slurm
+    environment:
+      CONTROL_MACHINE: controller
+      ACCOUNTING_STORAGE_HOST: database
+      COMPUTE_NODES: worker01 worker02
+      NFS_SERVER: nfs-server
+      NFS_SERVER_DIRS: '/nfs/secret:/nfs/home:/nfs/modules:/nfs/modulefiles'
+      NFS_CLIENT_DIRS: '/.secret:/home:/opt/apps/Linux:/opt/apps/modulefiles/Linux'
+
+networks:
+  slurm:
+
+volumes:
+  nfs-vol:
+    external: true
diff --git a/nfs-server/Dockerfile b/nfs-server/Dockerfile
new file mode 100644
index 0000000..713fd8c
--- /dev/null
+++ b/nfs-server/Dockerfile
@@ -0,0 +1,17 @@
+FROM krallin/centos-tini:7
+MAINTAINER Michael J. Stealey <michael.j.stealey@gmail.com>
+
+RUN rpmkeys --import file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 && \
+    yum -y --setopt=tsflags=nodocs install nfs-utils && \
+    mkdir -p /nfs && \
+    yum clean all
+
+ENV RPCNFSDCOUNT=8 \
+  NFS_SERVER_DIRS='/nfs/share'
+
+COPY docker-entrypoint.sh /docker-entrypoint.sh
+
+VOLUME ["/nfs"]
+EXPOSE 111/udp 2049/tcp
+
+ENTRYPOINT ["/usr/local/bin/tini", "--", "/docker-entrypoint.sh"]
diff --git a/nfs-server/docker-entrypoint.sh b/nfs-server/docker-entrypoint.sh
new file mode 100755
index 0000000..13729b7
--- /dev/null
+++ b/nfs-server/docker-entrypoint.sh
@@ -0,0 +1,93 @@
+#!/bin/bash
+set -e
+
+_start_nfs() {
+  exportfs -a
+  rpcbind
+  rpc.statd
+  rpc.nfsd
+  rpc.mountd
+}
+
+_nfs_server_mounts() {
+  IFS=':' read -r -a MNT_SERVER_ARRAY <<< "$NFS_SERVER_DIRS"
+  for server_mnt in "${MNT_SERVER_ARRAY[@]}"; do
+    if [[ ! -d $server_mnt ]]; then
+        mkdir -p $server_mnt
+    fi
+    chmod -R 777 $server_mnt
+    cat >> /etc/exports <<EOF
+${server_mnt} *(rw,sync,no_subtree_check,no_root_squash,fsid=$(( ( RANDOM % 100 )  + 200 )))
+EOF
+  done
+  cat /etc/exports
+}
+
+_sysconfig_nfs() {
+  cat > /etc/sysconfig/nfs <<EOF
+#
+#
+# To set lockd kernel module parameters please see
+#  /etc/modprobe.d/lockd.conf
+#
+
+# Optional arguments passed to rpc.nfsd. See rpc.nfsd(8)
+RPCNFSDARGS=""
+# Number of nfs server processes to be started.
+# The default is 8.
+RPCNFSDCOUNT=${RPCNFSDCOUNT}
+#
+# Set V4 grace period in seconds
+#NFSD_V4_GRACE=90
+#
+# Set V4 lease period in seconds
+#NFSD_V4_LEASE=90
+#
+# Optional arguments passed to rpc.mountd. See rpc.mountd(8)
+RPCMOUNTDOPTS=""
+# Port rpc.mountd should listen on.
+#MOUNTD_PORT=892
+#
+# Optional arguments passed to rpc.statd. See rpc.statd(8)
+STATDARG=""
+# Port rpc.statd should listen on.
+#STATD_PORT=662
+# Outgoing port statd should used. The default is port
+# is random
+#STATD_OUTGOING_PORT=2020
+# Specify callout program
+#STATD_HA_CALLOUT="/usr/local/bin/foo"
+#
+#
+# Optional arguments passed to sm-notify. See sm-notify(8)
+SMNOTIFYARGS=""
+#
+# Optional arguments passed to rpc.idmapd. See rpc.idmapd(8)
+RPCIDMAPDARGS=""
+#
+# Optional arguments passed to rpc.gssd. See rpc.gssd(8)
+# Note: The rpc-gssd service will not start unless the
+#       file /etc/krb5.keytab exists. If an alternate
+#       keytab is needed, that separate keytab file
+#       location may be defined in the rpc-gssd.service's
+#       systemd unit file under the ConditionPathExists
+#       parameter
+RPCGSSDARGS=""
+#
+# Enable usage of gssproxy. See gssproxy-mech(8).
+GSS_USE_PROXY="yes"
+#
+# Optional arguments passed to blkmapd. See blkmapd(8)
+BLKMAPDARGS=""
+EOF
+}
+
+### main ###
+_sysconfig_nfs
+_nfs_server_mounts
+_start_nfs
+
+rpcinfo -p
+showmount -e
+
+tail -f /dev/null
diff --git a/worker/Dockerfile b/worker/Dockerfile
index a1b7a5b..520cb66 100644
--- a/worker/Dockerfile
+++ b/worker/Dockerfile
@@ -28,6 +28,10 @@ RUN ./configure --prefix=/opt/apps \
 
 WORKDIR /home/worker
 
+ENV NFS_SERVER=nfs-server \
+  NFS_SERVER_DIRS=/nfs/secret:/nfs/home:/nfs/modules:/nfs/modulefiles \
+  NFS_CLIENT_DIRS=/secret:/home:/opt/apps/Linux:/opt/apps/modulefiles/Linux
+
 # clean up
 RUN rm -f /packages/slurm-*.rpm /packages/openmpi-*.rpm \
   && yum clean all \
diff --git a/worker/docker-entrypoint.sh b/worker/docker-entrypoint.sh
index f18b9f7..0bba977 100755
--- a/worker/docker-entrypoint.sh
+++ b/worker/docker-entrypoint.sh
@@ -1,6 +1,39 @@
 #!/usr/bin/env bash
 set -e
 
+_init_fstab() {
+  if [[ ! -f /etc/fstab ]]; then
+    cat > /etc/fstab << EOF
+### <server>:</remote/export> </local/directory> <nfs-type> <options> 0 0
+EOF
+  fi
+}
+
+_export_nfs_mounts() {
+  IFS=':' read -r -a MNT_SERVER_ARRAY <<< "$NFS_SERVER_DIRS"
+  IFS=':' read -r -a MNT_CLIENT_ARRAY <<< "$NFS_CLIENT_DIRS"
+  for i in "${!MNT_CLIENT_ARRAY[@]}"; do
+    if [[ ! -d ${MNT_CLIENT_ARRAY[$i]} ]]; then
+      mkdir -p ${MNT_CLIENT_ARRAY[$i]}
+    fi
+    cat >> /etc/fstab <<EOF
+${NFS_SERVER}:${MNT_SERVER_ARRAY[$i]} ${MNT_CLIENT_ARRAY[$i]} nfs rw,hard,intr 0 0
+EOF
+  done
+  cat /etc/fstab
+}
+
+_start_nfs_services() {
+  rpcbind
+  rpc.nfsd
+  sleep 2
+  echo "connecting to ${NFS_SERVER}"
+  until [ $(ping ${NFS_SERVER} -c 3 2>&1 >/dev/null)$? ]; do
+    echo -n "."
+    sleep 2
+  done
+}
+
 # start sshd server
 _sshd_host() {
   if [ ! -d /var/run/sshd ]; then
@@ -61,6 +94,14 @@ _slurmd() {
 }
 
 ### main ###
+_start_nfs_services
+_init_fstab
+_export_nfs_mounts
+mount -a
+
+rpcinfo -p $NFS_SERVER
+showmount -e $NFS_SERVER
+
 _sshd_host
 _munge_start_using_key
 _wait_for_worker