RABSW-637: DWS: Deploy to DP0

- Update to latest nnf-ec
- Disable --mock in config/manager/manager.yaml to run on DP0
- add script to set $VERSION for a developer build

setDevVersion.sh now locates the tag in artifactory that matches the current commit's SHA.

.gitignore

@@ -31,3 +31,6 @@ commands.log
 kind-config.yaml
 standalone-playground
 nnf-sos
+
+# .vscode log files
+.vscode/*.log

Jenkinsfile

@@ -1,16 +1,18 @@
-// Procedure for building NNF Element Controller  
+// Procedure for building NNF Storage Orchestration Services
 
 @Library('dst-shared@master') _
 
+// See https://github.hpe.com/hpe/hpc-dst-jenkins-shared-library for all
+// the inputs to the dockerBuildPipeline.
+// In particular: vars/dockerBuildPipeline.groovy
 dockerBuildPipeline {
         repository = "cray"
         imagePrefix = "cray"
         app = "dp-nnf-sos"
         name = "dp-nnf-sos"
         description = "Near Node Flash Storage Orchestration Services"
         dockerfile = "Dockerfile"
-        useLazyDocker = true
         autoJira = false
         createSDPManifest = false
-        product = "kj"
+        product = "rabsw"
 }

Makefile

@@ -3,7 +3,7 @@
 # To re-generate a bundle for another specific version without changing the standard setup, you can:
 # - use the VERSION as arg of the bundle target (e.g make bundle VERSION=0.0.2)
 # - use environment variables to overwrite this value (e.g export VERSION=0.0.2)
-VERSION=$(shell sed 1q .version)
+VERSION ?= $(shell sed 1q .version)
 
 # CHANNELS define the bundle channels used in the bundle.
 # Add a new line here if you would like to change its default config. (E.g CHANNELS = "preview,fast,stable")
@@ -24,28 +24,58 @@ BUNDLE_DEFAULT_CHANNEL := --default-channel=$(DEFAULT_CHANNEL)
 endif
 BUNDLE_METADATA_OPTS ?= $(BUNDLE_CHANNELS) $(BUNDLE_DEFAULT_CHANNEL)
 
+# GIT_TAG is the SHA of the current commit
+GIT_TAG=$(shell git rev-parse --short HEAD)
+
 # IMAGE_TAG_BASE defines the docker.io namespace and part of the image name for remote images.
 # This variable is used to construct full image tags for bundle and catalog images.
 #
 # For example, running 'make bundle-build bundle-push catalog-build catalog-push' will build and push both
 # cray.com/nnf-sos-bundle:$VERSION and cray.com/nnf-sos-catalog:$VERSION.
-IMAGE_TAG_BASE ?= arti.dev.cray.com/kj-docker-master-local/cray-dp-nnf-sos
+IMAGE_TAG_BASE ?= arti.dev.cray.com/rabsw-docker-master-local/cray-dp-nnf-sos
 
 # BUNDLE_IMG defines the image:tag used for the bundle.
 # You can use it as an arg. (E.g make bundle-build BUNDLE_IMG=<some-registry>/<project-name-bundle>:<tag>)
 BUNDLE_IMG ?= $(IMAGE_TAG_BASE)-bundle:v$(VERSION)
 
 # Image URL to use all building/pushing image targets
-#IMG ?= controller:latest
 IMG ?= $(IMAGE_TAG_BASE):$(VERSION)
 
+# Jenkins behaviors
+# pipeline_service builds its target docker image and stores it into 1 of 3 destination folders.
+# The behavior of where pipeline_service puts a build is dictated by name of the branch Jenkins
+# is building. (See https://github.hpe.com/hpe/hpc-dst-jenkins-shared-library/vars/getArtiRepository.groovy)
+#
+#         arti.dev.cray.com folder                                  Contents
+# ---------------------------------------------  -----------------------------------------------------
+# arti.dev.cray.com/rabsw-docker-master-local    master branch builds
+# arti.dev.cray.com/rabsw-docker-stable-local    release branch builds
+# arti.dev.cray.com/rabsw-docker-unstable-local  non-master && non-release branches, i.e. everything else
+#
+# pipeline_service tags the build with the following tag:
+#
+#   VERSION = sh(returnStdout: true, script: "cat .version").trim()   // .version file here
+#   def buildDate = new Date().format( 'yyyyMMddHHmmss' )
+#   BUILD_DATE = "${buildDate}"
+#   GIT_TAG = sh(returnStdout: true, script: "git rev-parse --short HEAD").trim()
+#
+#   IMAGE_TAG = getDockerImageTag(version: "${VERSION}", buildDate: "${BUILD_DATE}", gitTag: "${GIT_TAG}", gitBranch: "${GIT_BRANCH}")
+#
+# Because of the build date stamp, this tag is a bit difficult to use.
+# NOTE: master-local and stable-local have a 'latest' tag that can be used to fetch the latest of either
+#       the master or release branch.
+
 # Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
 CRD_OPTIONS ?= "crd:trivialVersions=true,preserveUnknownFields=false"
 
 # Tell Kustomize to deploy the default config, or an overlay.
 # To use the 'craystack' overlay:
 #   export KUBECONFIG=/my/craystack/kubeconfig.file
 #   make deploy OVERLAY=craystack
+#
+# To use the 'dp0' overlay:
+#   export KUBECONFIG=/my/dp0/kubeconfig.file
+#   make deploy OVERLAY=dp0
 OVERLAY ?= top
 
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)

config/crd/bases/crd.go

@@ -1,3 +1,3 @@
 package crd
 
-// Empty file so other repositories can import the custom resource definitionss
+// Empty file so other repositories can import the custom resource definitions

config/dp0/kustomization.yaml

@@ -0,0 +1,6 @@
+bases:
+- ../top
+
+patchesStrategicMerge:
+# Let the node-manager daemonset mount host dirs for lustre tools and libs.
+- manager_volumes_patch.yaml

config/dp0/manager_volumes_patch.yaml

@@ -0,0 +1,52 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: node-manager
+  namespace: system
+spec:
+  template:
+    spec:
+      containers:
+      - name: manager
+        args:
+          - --controller=node
+        volumeMounts:
+          - mountPath: /mnt
+            name: mnt-dir
+            mountPropagation: Bidirectional
+          - mountPath: /dev
+            name: dev-dir
+            mountPropagation: HostToContainer
+          - mountPath: /var/run/dbus
+            name: dbus-dir
+          - mountPath: /run/udev
+            name: udev-dir
+          - mountPath: /usr/sbin
+            name: usbin-dir
+          - mountPath: /usr/lib64
+            name: lib64-dir
+          - mountPath: /sys
+            name: sys-dir
+            mountPropagation: HostToContainer
+      volumes:
+        - name: mnt-dir
+          hostPath:
+            path: /mnt
+        - name: dev-dir
+          hostPath:
+            path: /dev
+        - name: dbus-dir
+          hostPath:
+            path: /var/run/dbus
+        - name: usbin-dir
+          hostPath:
+            path: /usr/sbin
+        - name: lib64-dir
+          hostPath:
+            path: /usr/lib64
+        - name: sys-dir
+          hostPath:
+            path: /sys
+        - name: udev-dir
+          hostPath:
+            path: /run/udev

config/manager/kustomization.yaml

@@ -12,5 +12,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 images:
 - name: controller
-  newName: arti.dev.cray.com/kj-docker-master-local/cray-dp-nnf-sos
+  newName: arti.dev.cray.com/rabsw-docker-master-local/cray-dp-nnf-sos
   newTag: 0.0.1

config/samples/scripts/wfrLustre

@@ -25,7 +25,7 @@ data+='    "wlmID": "5f239bd8-30db-450b-8c2c-a1a7c8631a1a"'
 data+='  }'
 data+='}'
 
-echo Attempting to create $rname ...
+echo Attempting to create $rname on "$DWSHOST":"$DWSPORT"
 jsondata="$data"
 
 admitResponse=$(curl -s -X POST "http://${DWSHOST}:${DWSPORT}/apis/dws.cray.hpe.com/v1alpha1/namespaces/$NAMESPACE/workflows" -H "Content-Type: application/json" -d "$jsondata")

config/samples/scripts/wfrRaw

@@ -9,26 +9,26 @@ rname=$1
 
 data='{'
 data+='  "metadata": {'
-data+='    "name": "##RNAME##",'
-data+='    "namespace": "##NAMESPACE##"'
+data+='    "name": "'$rname'",'
+data+='    "namespace": "'$NAMESPACE'"'
 data+='  },'
 data+='  "kind": "Workflow",'
 data+='  "apiVersion": "dws.cray.hpe.com/v1alpha1",'
 data+='  "spec": {'
 data+='    "desiredState": "proposal",'
 data+='    "dwDirectives": ['
-data+='      "#DW jobdw type=raw capacity=10TB name=ajf"'
+data+='      "#DW jobdw type=raw capacity=1GB name='$rname'"'
 data+='    ],'
 data+='    "jobID": 9000001,'
 data+='    "userID": 1001,'
 data+='    "wlmID": "5f239bd8-30db-450b-8c2c-a1a7c8631a1a"'
 data+='  }'
 data+='}'
 
-echo Attempting to create $rname ...
-jsondata=$(echo $data | sed -e s:##RNAME##:$rname:g -e s:##NAMESPACE##:"$NAMESPACE":g)
+echo Attempting to create $rname on "$DWSHOST":"$DWSPORT"
+jsondata="$data"
 
-admitResponse=$(curl -X POST "http://${DWSHOST}:${DWSPORT}/apis/dws.cray.hpe.com/v1alpha1/namespaces/$NAMESPACE/workflows" -H "Content-Type: application/json" -d "$jsondata")
+admitResponse=$(curl -s -X POST "http://${DWSHOST}:${DWSPORT}/apis/dws.cray.hpe.com/v1alpha1/namespaces/$NAMESPACE/workflows" -H "Content-Type: application/json" -d "$jsondata")
 retCode=$?
 
 # If the curl command failed, then we assume we failed to create the WFR...
@@ -37,12 +37,6 @@ if [ $retCode -ne 0 ]; then
     exit $retCode
 fi
 
-# curl command successful, look at the returned JSON to see if we were successful
-echo "$admitResponse" | \
-jq 'if .kind == "Status"
-	then
-		"Failed to create: $admitResponse"
-	else
-		.metadata.name, .
-	end, .'
-exit $retCode
+# Verify the resource was created.
+kubectl get -n default workflow "$rname"
+exit $?

go.mod

@@ -10,8 +10,8 @@ require (
 	github.com/go-logr/logr v0.4.0
 	github.com/onsi/ginkgo v1.16.5
 	github.com/onsi/gomega v1.17.0
-	github.hpe.com/hpe/hpc-dpm-dws-operator v0.0.0-20211213160018-2b73fb1030f9
-	github.hpe.com/hpe/hpc-rabsw-nnf-ec v1.0.6-0.20211213185652-ece897e5fe46
+	github.hpe.com/hpe/hpc-dpm-dws-operator v0.0.0-20220110213219-fcd5783304c2
+	github.hpe.com/hpe/hpc-rabsw-nnf-ec v1.0.6-0.20220107171404-8b3a8cdbd9de
 	k8s.io/api v0.22.2
 	k8s.io/apimachinery v0.22.2
 	k8s.io/client-go v0.22.2

go.sum

@@ -669,6 +669,10 @@ github.hpe.com/hpe/hpc-rabsw-nnf-ec v1.0.6-0.20211213150822-62de995dac76 h1:8+gz
 github.hpe.com/hpe/hpc-rabsw-nnf-ec v1.0.6-0.20211213150822-62de995dac76/go.mod h1:KTZsXDcX0VvrW79hC9eL3a3wTHfnnBD03X8IV/5+k0k=
 github.hpe.com/hpe/hpc-rabsw-nnf-ec v1.0.6-0.20211213185652-ece897e5fe46 h1:FNiN76QwONYRtNhTsUQkuJdNJrM03P2T+K+lWcKlatE=
 github.hpe.com/hpe/hpc-rabsw-nnf-ec v1.0.6-0.20211213185652-ece897e5fe46/go.mod h1:KTZsXDcX0VvrW79hC9eL3a3wTHfnnBD03X8IV/5+k0k=
+github.hpe.com/hpe/hpc-rabsw-nnf-ec v1.0.6-0.20211214153118-c080d8e8f741 h1:jIUZGJHK8/nbg/QfILKnQWDAKqt8fq8G75tiXZkaYsU=
+github.hpe.com/hpe/hpc-rabsw-nnf-ec v1.0.6-0.20211214153118-c080d8e8f741/go.mod h1:KTZsXDcX0VvrW79hC9eL3a3wTHfnnBD03X8IV/5+k0k=
+github.hpe.com/hpe/hpc-rabsw-nnf-ec v1.0.6-0.20220107171404-8b3a8cdbd9de h1:ewkY6kL1sIjb0FXGCGIUFHCtiCYkyWG5Ey/QfBIIXnU=
+github.hpe.com/hpe/hpc-rabsw-nnf-ec v1.0.6-0.20220107171404-8b3a8cdbd9de/go.mod h1:KTZsXDcX0VvrW79hC9eL3a3wTHfnnBD03X8IV/5+k0k=
 go.chromium.org/luci v0.0.0-20210915061045-7722a2154c29 h1:TgvWIQO0I0E7pKVLq/ARDhT2p2kHGKLtZhAN0EtUxiI=
 go.chromium.org/luci v0.0.0-20210915061045-7722a2154c29/go.mod h1:gk1qa1CBZAdya1PqugRJjkwk8LIymAoB5wf5UA7NrFk=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=

playground.sh

@@ -47,7 +47,7 @@ EOF
     # Taint the kind workers as rabbit nodes for the NLCMs, to keep any
     # non-NLCM pods off of them.
     NODES=$(kubectl get nodes --no-headers -o custom-columns=:metadata.name | grep -v control-plane | paste -d" " -s -)
-    kubectl taint nodes $NODES cray.nnf.node=true:NoSchedule
+    kubectl taint nodes "$NODES" cray.nnf.node=true:NoSchedule
 
     # Label the kind-workers as rabbit nodes for the NLCMs.
     for NODE in $(kubectl get nodes --no-headers | grep --invert-match "control-plane" | awk '{print $1}'); do
@@ -68,6 +68,42 @@ if [[ "$CMD" == kind-reset ]]; then
     ./playground.sh kind-create
 fi
 
+
+# The following commands apply to initializing the current DP0 environment
+# Nodes containing 'cn' are considered to be worker nodes for the time being.
+if [[ "$CMD" == dp0-init ]]; then
+    COMPUTE_NODES=$(kubectl get nodes --no-headers -o custom-columns=:metadata.name | grep cn | paste -d" " -s -)
+    RABBIT_NODES=$(kubectl get nodes --no-headers -o custom-columns=:metadata.name | grep -v cn | grep -v master | paste -d" " -s -)
+    MASTER_NODES=$(kubectl get nodes --no-headers -o custom-columns=:metadata.name | grep master | paste -d" " -s -)
+
+    echo COMPUTE_NODES "$COMPUTE_NODES"
+    echo RABBIT_NODES "$RABBIT_NODES"
+    echo MASTER_NODES "$MASTER_NODES"
+
+    # Label the COMPUTE_NODES to allow them to handle wlm and nnf-sos
+    # We are using COMPUTE_NODES as generic k8s workers
+    for NODE in $COMPUTE_NODES; do
+        # Label them for SLCMs.
+        kubectl label node "$NODE" cray.nnf.manager=true
+        kubectl label node "$NODE" cray.wlm.manager=true
+    done
+
+    for NODE in $RABBIT_NODES; do
+        # Taint the rabbit nodes for the NLCMs, to keep any
+        # non-NLCM pods off of them.
+        kubectl taint node "$NODE" cray.nnf.node=true:NoSchedule
+
+        # Label the rabbit nodes for the NLCMs.
+        kubectl label node "$NODE" cray.nnf.node=true
+        kubectl label node "$NODE" cray.nnf.x-name="$NODE"
+    done
+
+    #Required for webhooks
+    # kubectl delete -f https://github.com/jetstack/cert-manager/releases/download/v1.4.0/cert-manager.yaml
+    kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.4.0/cert-manager.yaml
+fi
+
+
 if [[ "$CMD" == restart-pods ]]; then
     ./playground.sh pause all
     while :
@@ -229,7 +265,7 @@ then
     mkdir $PLAY
     mkdir $DWSPLAY
     cp -r .dws-operator/config $DWSPLAY
-    cp .dws-operator/Makefile $DWSPLAY
+    cp .dws-operator/Makefile .dws-operator/.version $DWSPLAY
     cp -r config $PLAY
     cp .version Makefile playground.sh $PLAY
 fi

setDevVersion.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+# Use source to push the environment variables back into the calling shell
+# "source ./setDevVersion.sh"
+
+# Command to access arti and list all of the docker images there filtering out the tag for the
+# version matching the current git SHA
+unset VERSION
+unset IMAGE_TAG_BASE
+
+# Setup some artifactory paths to the developer and master branch locations
+ARTI_URL_DEV=https://arti.dev.cray.com/artifactory/rabsw-docker-unstable-local/cray-dp-nnf-sos/
+ARTI_URL_MASTER=https://arti.dev.cray.com/artifactory/rabsw-docker-master-local/cray-dp-nnf-sos/
+
+# Retrieve the name of the current branch. If we are in detached HEAD state, assume it is master.
+CURRENT_BRANCH_NAME=$(git rev-parse --abbrev-ref HEAD)
+
+# Depending on whether we are on the master branch or not, setup for deployment from artifactory
+# NOTE: Detached HEAD state is assumed to match 'master'
+if [[ "$CURRENT_BRANCH_NAME" == "master" ]] || [[ "$CURRENT_BRANCH_NAME" == "HEAD" ]]; then
+    ARTI_URL="$ARTI_URL_MASTER"
+else    # not on the master branch
+    ARTI_URL="$ARTI_URL_DEV"
+
+    # Deploying a developer build requires the IMAGE_TAG_BASE to change as well.
+    # Master branch is the default, so we don't change it when we are on the master branch.
+    IMAGE_TAG_BASE=arti.dev.cray.com/rabsw-docker-unstable-local/cray-dp-nnf-sos
+    export IMAGE_TAG_BASE
+    echo IMAGE_TAG_BASE: "$IMAGE_TAG_BASE"
+fi
+
+# Locate the container tags in arti to set the VERSION environment variable
+# which allows us to run `make deploy` and pull the correct version from ARTI.
+LATEST_LOCAL_COMMIT=$(git rev-parse --short HEAD)
+ARTI_TAG=$(wget --spider --recursive --no-parent -l1 "$ARTI_URL" 2>&1 | grep -- ^-- | awk '{print $3}' | grep "$LATEST_LOCAL_COMMIT" | tail -1)
+VERSION=$(basename "$ARTI_TAG")
+export VERSION
+echo VERSION: "$VERSION"