Merge pull request #23 in RABSW/nnf-sos from feature/test to master

Squashed commit of the following:

commit 1b144fd8546d29a61fa7f2df911323663e5b1b8d
Author: Bill Johnson <[email protected]>
Date:   Mon Sep 13 13:01:54 2021 -0500

    RABSW-514: Removed comment block in SetupReconciler

commit 898537dcf67d581896da5387bc4d64b825f6427f
Author: Bill Johnson <[email protected]>
Date:   Mon Sep 13 08:24:47 2021 -0500

    RABSW-514: Removed redundant WORKSPACE from testing stage

commit 6297579ebb758b3c3c868d7387008d539b9f913f
Author: Bill Johnson <[email protected]>
Date:   Fri Sep 10 15:11:41 2021 -0500

    RABSW-514: Refactored Dockerfile according to pull request feedback

commit 5c7a58bfd023f10c7c3773675894d29ffac0a0f7
Author: Bill Johnson <[email protected]>
Date:   Thu Sep 9 17:28:26 2021 -0500

    RABSW-514: Removed cobertura stuff as it's not used by jenkins and is fighting with vendoring

commit 9e338220a35199dbf39feb876b0afcc28b1dc15c
Author: Bill Johnson <[email protected]>
Date:   Thu Sep 9 16:46:21 2021 -0500

    RABSW-514: Fixed production stage in Dockerfile to run successfully

commit 99f5602b00e2ef11a1d3c645a8ffa4e353d824a0
Author: Bill Johnson <[email protected]>
Date:   Thu Sep 9 13:50:14 2021 -0500

    RABSW-514: Reduced the number of RUN statements in Dockerfile

commit 3ea46aca8572edb728b13a1596cf9e6df3088157
Merge: 5329885 c4731c3
Author: Bill Johnson <[email protected]>
Date:   Thu Sep 9 13:08:12 2021 -0500

    Merged master, refactored Dockerfile for lustre and unit tests

commit 5329885be9e3334d21dd8a9d92ca97355496edb2
Author: Bill Johnson <[email protected]>
Date:   Wed Sep 8 13:21:02 2021 -0500

    Cleaned up dockerfile, added 2 run scripts

commit 87c89d2ee8ec4d67b1e7b51126b7713c8f6792b3
Author: Bill Johnson <[email protected]>
Date:   Thu Aug 26 15:26:47 2021 -0500

    RABSW-514: Fixed bad .dwsoperator branch in .gitmodules

commit 06458342c99480101db97c6e5c59cac404f87f9e
Author: Nate Roiger <[email protected]>
Date:   Thu Aug 12 07:48:19 2021 -0500

    Initial round of integration testing

.dockerignore

@@ -3,3 +3,6 @@
 !**/*.go
 !**/*.mod
 !**/*.sum
+Dockerfile
+.git
+.vscode

.gitmodules

@@ -0,0 +1,4 @@
+[submodule ".dws-operator"]
+	path = .dws-operator
+	url = ssh://[email protected]:7999/dpm/dws-operator
+	branch = master

.vscode/launch.json

@@ -0,0 +1,20 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Test Package",
+            "type": "go",
+            "request": "launch",
+            "mode": "test",
+            "program": "${workspaceFolder}",
+            "args": ["./..."],
+            "env": {
+                "KUBEBUILDER_ASSETS": "${workspaceFolder}/testbin/bin"
+            },
+
+        }
+    ]
+}

Dockerfile

@@ -3,10 +3,9 @@ FROM arti.dev.cray.com/baseos-docker-master-local/centos:centos7 AS base
 WORKDIR /
 
 # Install basic dependencies
-RUN yum install -y gzip wget gcc tar https://packages.endpoint.com/rhel/7/os/x86_64/endpoint-repo-1.7-1.x86_64.rpm
+RUN yum install -y make git gzip wget gcc tar https://packages.endpoint.com/rhel/7/os/x86_64/endpoint-repo-1.7-1.x86_64.rpm
 
 # Retrieve lustre-rpms
-RUN mkdir -p /tmp/lustre-rpms
 WORKDIR /tmp/lustre-rpms
 
 RUN wget https://downloads.whamcloud.com/public/e2fsprogs/1.45.6.wc1/el7/RPMS/x86_64/e2fsprogs-1.45.6.wc1-0.el7.x86_64.rpm \
@@ -54,6 +53,7 @@ ENV GOROOT="/go"
 ENV PATH="${PATH}:${GOROOT}/bin" GOPRIVATE="stash.us.cray.com"
 
 WORKDIR /workspace
+
 # Copy the Go Modules manifests
 COPY go.mod go.mod
 COPY go.sum go.sum
@@ -69,6 +69,20 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o manager main.go
 
 ENTRYPOINT ["/bin/sh"]
 
+FROM builder as testing
+WORKDIR /workspace
+
+COPY .dws-operator/ .dws-operator/
+COPY hack/ hack/
+COPY runContainerTest.sh .
+COPY Makefile .
+
+RUN go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest && \
+    make manifests && make generate && make fmt &&  make vet && \
+    mkdir -p /workspace/testbin && /bin/bash -c "test -f /workspace/testbin/setup-envtest.sh || curl -sSLo /workspace/testbin/setup-envtest.sh https://raw.githubusercontent.com/kubernetes-sigs/controller-runtime/v0.7.2/hack/setup-envtest.sh" && \
+    /bin/bash -c "source /workspace/testbin/setup-envtest.sh; fetch_envtest_tools /workspace/testbin; setup_envtest_env /workspace/testbin"
+
+ENTRYPOINT ["sh", "/workspace/runContainerTest.sh"]
 
 # The final application stage.
 FROM application-base

Jenkinsfile

@@ -0,0 +1,16 @@
+// Procedure for building NNF Element Controller  
+
+@Library('dst-shared@master') _
+
+dockerBuildPipeline {
+        repository = "cray"
+        imagePrefix = "cray"
+        app = "dp-nnf-sos"
+        name = "dp-nnf-sos"
+        description = "Near Node Flash Storage Orchestration Services"
+        dockerfile = "Dockerfile"
+        useLazyDocker = true
+        autoJira = false
+        createSDPManifest = false
+        product = "kj"
+}

Makefile

@@ -87,6 +87,8 @@ fmt: ## Run go fmt against code.
 vet: ## Run go vet against code.
 	go vet ./...
 
+##@ Test
+
 ENVTEST_ASSETS_DIR=$(shell pwd)/testbin
 test: manifests generate fmt vet ## Run tests.
 	mkdir -p ${ENVTEST_ASSETS_DIR}

Readme.md

@@ -0,0 +1,22 @@
+# NNF Storage Orchestration Services (SOS)
+
+The NNF SOS project is a collection of Kubernetes Custom Resource Definitions (CRDs) and associated controllers that permit the creation, management, and destruction of storage on an NNF cluster.
+
+## Setup
+
+This project depends on additional CRDs present in the dws-operator project. It makes use of `git submodules` to make the dependecy in version control. To clone this project, use the additional `--recurse-submodules` option.
+```
+git clone --recurse-submodules ssh://[email protected]:7999/rabsw/nnf-sos.git
+```
+
+If you've already clone the repo, initialize the submodules with `git submodule init`
+
+## Building
+
+Run `make`
+
+## Testing
+
+Run `make test`. This ensures the proper utilities (kube-apiserver, kubectl, etcd) are installed in the `./testbin` directory
+
+If you use VS Code, the Test Package configuration exists to start a live debugger of the test suite.

controllers/integration_test.go

@@ -0,0 +1,102 @@
+/*
+Copyright 2021 Hewlett Packard Enterprise Development LP
+*/
+
+package controllers
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	nnfv1alpha1 "stash.us.cray.com/RABSW/nnf-sos/api/v1alpha1"
+	dwsv1alpha1 "stash.us.cray.com/dpm/dws-operator/api/v1alpha1"
+)
+
+var _ = Describe("Integration Test", func() {
+
+	const (
+		WorkflowName      = "test-workflow"
+		WorkflowNamespace = "default"
+		WorkflowID        = "test"
+	)
+
+	const timeout = time.Second * 10
+	const interval = time.Second * 1
+
+	Describe("Creating a Workflow Resource", func() {
+
+		for _, fs := range []string{"raw", "lvm", "xfs", "gfs2", "lustre"} {
+
+			// TODO: Remove this as more file systems are supported and tested
+			BeforeEach(func() {
+				if fs != "raw" {
+					Skip("File System Not Supported")
+				}
+			})
+
+			Context(fmt.Sprintf("%s File System", strings.ToTitle(fs)), func() {
+
+				var workflow *dwsv1alpha1.Workflow
+				var storage *nnfv1alpha1.NnfStorage
+
+				It("Should create successfully", func() {
+					workflow = &dwsv1alpha1.Workflow{
+						ObjectMeta: metav1.ObjectMeta{
+							Name:      WorkflowName,
+							Namespace: WorkflowNamespace,
+						},
+						Spec: dwsv1alpha1.WorkflowSpec{
+							DesiredState: "proposal", // TODO: This should be defined somewhere
+							WLMID:        WorkflowID,
+							DWDirectives: []string{
+								fmt.Sprintf("#DW name=test filesystem=%s capacity=10GB", fs), // TODO: Should be a MakeDW method
+							},
+						},
+					}
+
+					Expect(k8sClient.Create(context.Background(), workflow)).Should(Succeed())
+				})
+
+				It("Should reach ready status and proposed state", func() {
+					Eventually(func() bool {
+						k8sClient.Get(context.Background(), types.NamespacedName{Name: WorkflowName, Namespace: WorkflowNamespace}, workflow)
+						return workflow.Status.Ready
+					}, timeout, interval).Should(BeTrue())
+
+					Expect(workflow.Status.State).Should(Equal("proposal"))
+				})
+
+				PIt("Should have a single DW breakdown", func() {
+					Expect(workflow.Status.DWDirectiveBreakdowns).Should(HaveLen(1))
+				})
+
+				PIt("Should have a NNF Storage reference in the DW breakdown", func() {
+					//TODO
+				})
+
+				PIt("Should have created the storage allocation", func() {
+					err := k8sClient.Get(context.Background(), types.NamespacedName{Name: "", Namespace: ""}, storage)
+					Expect(err).ShouldNot(HaveOccurred())
+				})
+
+				PContext("Assigning NNF Node Names to NNF Storage", func() {
+
+					var nodes *nnfv1alpha1.NnfNodeList
+					It("Should find NNF Nodes", func() {
+						err := k8sClient.List(context.Background(), nodes, &client.ListOptions{})
+						Expect(err).ShouldNot(HaveOccurred())
+					})
+
+				})
+			})
+		}
+	})
+})

controllers/nnf_node_controller.go

@@ -57,6 +57,12 @@ type NnfNodeReconciler struct {
 func (r *NnfNodeReconciler) Start(ctx context.Context) error {
 	log := r.Log.WithValues("Node", r.NamespacedName, "State", "Start")
 
+	// During testing, the NNF Node Reconciler is started before the kubeapi-server runs, so any Get() will
+	// fail with 'connection refused'. The test code will instead bootstrap some nodes using the k8s test client.
+	if r.NamespacedName.String() == string(types.Separator) {
+		return nil
+	}
+
 	// Create a namespace unique to this node based on the node's x-name.
 	namespace := &corev1.Namespace{}
 	if err := r.Get(ctx, types.NamespacedName{Name: r.Namespace}, namespace); err != nil {

controllers/suite_test.go

@@ -8,6 +8,8 @@ import (
 	"path/filepath"
 	"testing"
 
+	ctrl "sigs.k8s.io/controller-runtime"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	"k8s.io/client-go/kubernetes/scheme"
@@ -19,6 +21,10 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 
 	nnfv1alpha1 "stash.us.cray.com/RABSW/nnf-sos/api/v1alpha1"
+
+	dwsv1alpha1 "stash.us.cray.com/dpm/dws-operator/api/v1alpha1"
+
+	dwsctrls "stash.us.cray.com/dpm/dws-operator/controllers"
 	//+kubebuilder:scaffold:imports
 )
 
@@ -29,6 +35,7 @@ var cfg *rest.Config
 var k8sClient client.Client
 var testEnv *envtest.Environment
 
+// This is the single entry point for Go test, and ensures all Ginkgo style test are run
 func TestAPIs(t *testing.T) {
 	RegisterFailHandler(Fail)
 
@@ -42,8 +49,12 @@ var _ = BeforeSuite(func() {
 
 	By("bootstrapping test environment")
 	testEnv = &envtest.Environment{
-		CRDDirectoryPaths:     []string{filepath.Join("..", "config", "crd", "bases")},
+		CRDDirectoryPaths: []string{
+			filepath.Join("..", "config", "crd", "bases"),
+			filepath.Join("..", ".dws-operator", "config", "crd", "bases"),
+		},
 		ErrorIfCRDPathMissing: true,
+		//WebhookInstallOptions: , If we want to test the webhooks, need to figure this out
 	}
 
 	cfg, err := testEnv.Start()
@@ -53,10 +64,7 @@ var _ = BeforeSuite(func() {
 	err = nnfv1alpha1.AddToScheme(scheme.Scheme)
 	Expect(err).NotTo(HaveOccurred())
 
-	err = nnfv1alpha1.AddToScheme(scheme.Scheme)
-	Expect(err).NotTo(HaveOccurred())
-
-	err = nnfv1alpha1.AddToScheme(scheme.Scheme)
+	err = dwsv1alpha1.AddToScheme(scheme.Scheme)
 	Expect(err).NotTo(HaveOccurred())
 
 	//+kubebuilder:scaffold:scheme
@@ -65,6 +73,57 @@ var _ = BeforeSuite(func() {
 	Expect(err).NotTo(HaveOccurred())
 	Expect(k8sClient).NotTo(BeNil())
 
+	k8sManager, err := ctrl.NewManager(cfg, ctrl.Options{
+		Scheme: scheme.Scheme,
+	})
+	Expect(err).NotTo(HaveOccurred())
+
+	/*
+		Start Everything
+	*/
+
+	//err = (&dwsv1alpha1.Workflow{}).SetupWebhookWithManager(k8sManager)
+	//Expect(err).ToNot(HaveOccurred())
+
+	err = (&dwsctrls.WorkflowReconciler{
+		Client: k8sManager.GetClient(),
+		Log:    ctrl.Log.WithName("controllers").WithName("Workflow"),
+	}).SetupWithManager(k8sManager)
+	Expect(err).ToNot(HaveOccurred())
+
+	err = (&WorkflowReconciler{
+		Client: k8sManager.GetClient(),
+		Log:    ctrl.Log.WithName("controllers").WithName("NnfWorkflow"),
+	}).SetupWithManager(k8sManager)
+	Expect(err).ToNot(HaveOccurred())
+
+	err = (&NnfNodeReconciler{
+		Client: k8sManager.GetClient(),
+		Log:    ctrl.Log.WithName("controllers").WithName("NnfNode"),
+	}).SetupWithManager(k8sManager)
+	Expect(err).ToNot(HaveOccurred())
+
+	err = (&NnfNodeStorageReconciler{
+		Client: k8sManager.GetClient(),
+		Log:    ctrl.Log.WithName("controllers").WithName("NnfNodeStorage"),
+	}).SetupWithManager(k8sManager)
+	Expect(err).ToNot(HaveOccurred())
+
+	err = (&NnfStorageReconciler{
+		Client: k8sManager.GetClient(),
+		Log:    ctrl.Log.WithName("controllers").WithName("NnfStorage"),
+	}).SetupWithManager(k8sManager)
+	Expect(err).ToNot(HaveOccurred())
+
+	go func() {
+		defer GinkgoRecover()
+		err = k8sManager.Start(ctrl.SetupSignalHandler())
+		Expect(err).NotTo(HaveOccurred())
+	}()
+
+	k8sClient = k8sManager.GetClient()
+	Expect(k8sClient).ToNot(BeNil())
+
 }, 60)
 
 var _ = AfterSuite(func() {

go.mod

@@ -16,3 +16,6 @@ require (
 	stash.us.cray.com/rabsw/nnf-ec v1.0.6-0.20210907162304-2d0676f76e6c
 	stash.us.cray.com/rabsw/rfsf-openapi v0.1.14-0.20210830201758-8dbd76f01e7d
 )
+
+replace stash.us.cray.com/dpm/dws-operator => ./.dws-operator
+

runContainerTest.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash 
+
+CWD=`pwd`
+echo "Running in $CWD, setting up envtest"
+
+export ENVTEST_ASSETS_DIR=/nnf/testbin
+mkdir -p ${ENVTEST_ASSETS_DIR}
+test -f ${ENVTEST_ASSETS_DIR}/setup-envtest.sh || curl -sSLo ${ENVTEST_ASSETS_DIR}/setup-envtest.sh https://raw.githubusercontent.com/kubernetes-sigs/controller-runtime/v0.7.2/hack/setup-envtest.sh
+source ${ENVTEST_ASSETS_DIR}/setup-envtest.sh
+fetch_envtest_tools ${ENVTEST_ASSETS_DIR}
+setup_envtest_env ${ENVTEST_ASSETS_DIR}
+
+echo Running unit tests
+
+go test ./... -coverprofile cover.out > results.txt
+cat results.txt
+
+grep FAIL results.txt && echo "Unit tests failure" && rm results.txt && exit 1 
+
+echo "Unit tests successful" && rm results.txt
+

runUnitTest.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+# Copyright 2021 Hewlett Packard Enterprise Development LP
+
+# This is a magic file, whos presence is an idication to DST pipeline that
+# this repository should be running tests.
+
+make docker-test