Merge pull request warewulf#934 from mslacken/FixKmod

Fix empty kernel modules and drivers

CHANGELOG.md

@@ -136,6 +136,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   has a container these binaries are extracted from the container image.
 - overlays from different profiles for one node are now merged, overlays can be
   excluded with ~ prefix and in listings are listed as !{excluded_profile} #885
+- modules are now correctly included
 
 ## [4.4.0] 2023-01-18
 

internal/app/wwctl/kernel/imprt/main.go

@@ -49,12 +49,12 @@ func CobraRunE(cmd *cobra.Command, args []string) error {
 	} else if OptDetect && (OptContainer != "") {
 		kernelName = OptContainer
 	}
-	output, err := kernel.Build(kernelVersion, kernelName, OptRoot)
+	err = kernel.Build(kernelVersion, kernelName, OptRoot)
 	if err != nil {
 		wwlog.Error("Failed building kernel: %s", err)
 		os.Exit(1)
 	} else {
-		fmt.Printf("%s: %s\n", kernelName, output)
+		fmt.Printf("%s: %s\n", kernelName, "Finished kernel build")
 	}
 
 	if SetDefault {

internal/pkg/kernel/kernel.go

@@ -111,9 +111,17 @@ func ListKernels() ([]string, error) {
 	return ret, nil
 }
 
-func Build(kernelVersion, kernelName, root string) (string, error) {
-	kernelDriversRelative := path.Join("/lib/modules/", kernelVersion)
-	kernelDrivers := path.Join(root, kernelDriversRelative)
+/*
+Triggers the kernel extraction and build of the modules for the given
+kernel version. A name for this kernel and were to find has also to be
+supplied
+*/
+func Build(kernelVersion, kernelName, root string) error {
+	kernelDrivers := []string{path.Join("lib/modules/",
+		kernelVersion, "*"),
+		"lib/firmware/*",
+		"lib/modprobe.d",
+		"lib/modules-load.d"}
 	kernelDestination := KernelImage(kernelName)
 	driversDestination := KmodsImage(kernelName)
 	versionDestination := KernelVersionFile(kernelName)
@@ -122,17 +130,17 @@ func Build(kernelVersion, kernelName, root string) (string, error) {
 	// Create the destination paths just in case it doesn't exist
 	err := os.MkdirAll(path.Dir(kernelDestination), 0755)
 	if err != nil {
-		return "", errors.Wrap(err, "failed to create kernel dest")
+		return errors.Wrap(err, "failed to create kernel dest")
 	}
 
 	err = os.MkdirAll(path.Dir(driversDestination), 0755)
 	if err != nil {
-		return "", errors.Wrap(err, "failed to create driver dest")
+		return errors.Wrap(err, "failed to create driver dest")
 	}
 
 	err = os.MkdirAll(path.Dir(versionDestination), 0755)
 	if err != nil {
-		return "", fmt.Errorf("failed to create version dest: %s", err)
+		return fmt.Errorf("failed to create version dest: %s", err)
 	}
 
 	for _, searchPath := range kernelSearchPaths {
@@ -146,20 +154,16 @@ func Build(kernelVersion, kernelName, root string) (string, error) {
 
 	if kernelSource == "" {
 		wwlog.Error("Could not locate kernel image")
-		return "", errors.New("could not locate kernel image")
+		return errors.New("could not locate kernel image")
 	} else {
 		wwlog.Info("Found kernel at: %s", kernelSource)
 	}
 
-	if !util.IsDir(kernelDrivers) {
-		return "", errors.New("Could not locate kernel drivers")
-	}
-
 	wwlog.Verbose("Setting up Kernel")
 	if _, err := os.Stat(kernelSource); err == nil {
 		kernel, err := os.Open(kernelSource)
 		if err != nil {
-			return "", errors.Wrap(err, "could not open kernel")
+			return errors.Wrap(err, "could not open kernel")
 		}
 		defer kernel.Close()
 
@@ -169,63 +173,59 @@ func Build(kernelVersion, kernelName, root string) (string, error) {
 
 			writer, err := os.Create(kernelDestination)
 			if err != nil {
-				return "", errors.Wrap(err, "could not decompress kernel")
+				return errors.Wrap(err, "could not decompress kernel")
 			}
 			defer writer.Close()
 
 			_, err = io.Copy(writer, gzipreader)
 			if err != nil {
-				return "", errors.Wrap(err, "could not write decompressed kernel")
+				return errors.Wrap(err, "could not write decompressed kernel")
 			}
 
 		} else {
 
 			err := util.CopyFile(kernelSource, kernelDestination)
 			if err != nil {
-				return "", errors.Wrap(err, "could not copy kernel")
+				return errors.Wrap(err, "could not copy kernel")
 			}
 		}
 
 	}
 
-	if _, err := os.Stat(kernelDrivers); err == nil {
-		name := kernelName + " drivers"
-		wwlog.Verbose("Creating image for %s: %s", name, root)
-
-		err = util.BuildFsImage(
-			name,
-			root,
-			driversDestination,
-			[]string{
-				"." + kernelDriversRelative,
-				"./lib/firmware"},
-			[]string{},
-			// ignore cross-device files
-			true,
-			"newc",
-			// dereference symbolic links
-			"-L")
+	name := kernelName + " drivers"
+	wwlog.Verbose("Creating image for %s: %s", name, root)
+
+	err = util.BuildFsImage(
+		name,
+		root,
+		driversDestination,
+		kernelDrivers,
+		[]string{},
+		// ignore cross-device files
+		true,
+		"newc",
+		// dereference symbolic links
+		"-L")
 
-		if err != nil {
-			return "", err
-		}
+	if err != nil {
+		return err
 	}
 
 	wwlog.Verbose("Creating version file")
 	file, err := os.Create(versionDestination)
 	if err != nil {
-		return "", errors.Wrap(err, "Failed to create version file")
+		return errors.Wrap(err, "Failed to create version file")
 	}
 	defer file.Close()
 	_, err = io.WriteString(file, kernelVersion)
 	if err != nil {
-		return "", errors.Wrap(err, "Could not write kernel version")
+		return errors.Wrap(err, "Could not write kernel version")
 	}
 	err = file.Sync()
 	if err != nil {
-		return "", errors.Wrap(err, "Could not sync kernel version")
+		return errors.Wrap(err, "Could not sync kernel version")
 	}
-	return "Done", nil
+	return nil
 }
 
 func DeleteKernel(name string) error {

internal/pkg/kernel/kernel_test.go

@@ -0,0 +1,72 @@
+package kernel
+
+import (
+	"os"
+	"path"
+	"testing"
+
+	warewulfconf "github.com/hpcng/warewulf/internal/pkg/config"
+	"github.com/hpcng/warewulf/internal/pkg/util"
+	"github.com/hpcng/warewulf/internal/pkg/wwlog"
+	"github.com/stretchr/testify/assert"
+)
+
+var kernelBuildTests = []struct {
+	kernelVersion  string
+	kernelName     string
+	kernelFileName string
+	succeed        bool
+}{
+	{"4.3.2.1", "kernel1", "vmlinuz-1.2.3.4.gz", false},
+	{"1.2.3.4", "kernel1", "vmlinuz-1.2.3.4.gz", true},
+}
+
+func Test_BuildKernel(t *testing.T) {
+	wwlog.SetLogLevel(wwlog.DEBUG)
+	srvDir, err := os.MkdirTemp(os.TempDir(), "ww-test-srv-*")
+	assert.NoError(t, err)
+	defer os.RemoveAll(srvDir)
+	conf := warewulfconf.Get()
+	conf.Paths.WWProvisiondir = srvDir
+	kernelDir, err := os.MkdirTemp(os.TempDir(), "ww-test-kernel-*")
+	assert.NoError(t, err)
+	defer os.RemoveAll(kernelDir)
+	{
+		err = os.MkdirAll(path.Join(kernelDir, "boot"), 0755)
+		assert.NoError(t, err)
+		err = os.MkdirAll(path.Join(kernelDir, "lib/modules/old-kernel"), 0755)
+		assert.NoError(t, err)
+		_, err = os.Create(path.Join(kernelDir, "lib/modules/old-kernel/old-module"))
+		assert.NoError(t, err)
+		err = os.MkdirAll(path.Join(kernelDir, "lib/firmware"), 0755)
+		assert.NoError(t, err)
+		_, err = os.Create(path.Join(kernelDir, "lib/firmware/test-firmware"))
+		assert.NoError(t, err)
+		for _, tt := range kernelBuildTests {
+			_, err = os.Create(path.Join(kernelDir, "boot", tt.kernelFileName))
+			assert.NoError(t, err)
+			err = os.MkdirAll(path.Join(kernelDir, "lib/modules", tt.kernelVersion, "/nested"), 0755)
+			assert.NoError(t, err)
+			_, err = os.Create(path.Join(kernelDir, "lib/modules", tt.kernelVersion, "test-module"))
+			assert.NoError(t, err)
+			err = os.Symlink(path.Join(kernelDir, "lib/modules/old-kernel/old-module"), path.Join(kernelDir, "lib/modules", tt.kernelVersion, "symlink-module"))
+			assert.NoError(t, err)
+		}
+	}
+	for _, tt := range kernelBuildTests {
+		t.Run(tt.kernelName, func(t *testing.T) {
+			err = Build(tt.kernelVersion, tt.kernelName, kernelDir)
+			if tt.succeed {
+				assert.NoError(t, err)
+				assert.FileExists(t, path.Join(srvDir, "kernel", tt.kernelName, "vmlinuz"))
+				assert.FileExists(t, path.Join(srvDir, "kernel", tt.kernelName, "kmods.img.gz"))
+				assert.FileExists(t, path.Join(srvDir, "kernel", tt.kernelName, "kmods.img"))
+				files, err := util.CpioFiles(path.Join(srvDir, "kernel", tt.kernelName, "kmods.img"))
+				assert.NoError(t, err)
+				assert.ElementsMatch(t, files, []string{"lib/firmware/test-firmware", "lib/modules/1.2.3.4/symlink-module", "lib/modules/1.2.3.4/test-module", "lib/modules/1.2.3.4/nested"})
+			} else {
+				assert.Error(t, err)
+			}
+		})
+	}
+}

internal/pkg/overlay/overlay_test.go

@@ -1,15 +1,17 @@
 package overlay
 
 import (
-	warewulfconf "github.com/hpcng/warewulf/internal/pkg/config"
-	"github.com/hpcng/warewulf/internal/pkg/node"
-	"github.com/sassoftware/go-rpmutils/cpio"
-	"github.com/stretchr/testify/assert"
 	"io"
 	"os"
 	"path"
 	"sort"
 	"testing"
+
+	warewulfconf "github.com/hpcng/warewulf/internal/pkg/config"
+	"github.com/hpcng/warewulf/internal/pkg/node"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/hpcng/warewulf/internal/pkg/util"
 )
 
 var buildOverlayTests = []struct {
@@ -161,9 +163,9 @@ func Test_BuildOverlay(t *testing.T) {
 			if tt.image != "" {
 				image := path.Join(provisionDir, "overlays", tt.image)
 				assert.FileExists(t, image)
-
 				sort.Strings(tt.contents)
-				files := cpioFiles(t, image)
+				files, err := util.CpioFiles(image)
+				assert.NoError(t, err)
 				sort.Strings(files)
 				assert.Equal(t, tt.contents, files)
 			} else {
@@ -401,20 +403,3 @@ func dirIsEmpty(t *testing.T, name string) bool {
 	t.Log(dirnames)
 	return false
 }
-
-func cpioFiles(t *testing.T, name string) (files []string) {
-	f, openErr := os.Open(name)
-	if openErr != nil {
-		return
-	}
-	defer f.Close()
-
-	reader := cpio.NewReader(f)
-	for {
-		header, err := reader.Next()
-		if err != nil {
-			return
-		}
-		files = append(files, header.Filename())
-	}
-}

internal/pkg/util/cpio.go

@@ -0,0 +1,31 @@
+package util
+
+import (
+	"io"
+	"os"
+
+	"github.com/sassoftware/go-rpmutils/cpio"
+)
+
+/*
+Opens cpio archive and returns the file list
+*/
+func CpioFiles(name string) (files []string, err error) {
+	f, err := os.Open(name)
+	if err != nil {
+		return files, err
+	}
+	defer f.Close()
+
+	reader := cpio.NewReader(f)
+	for {
+		header, err := reader.Next()
+		if err == io.EOF {
+			return files, nil
+		}
+		if err != nil {
+			return files, err
+		}
+		files = append(files, header.Filename())
+	}
+}