Adds support for creating Windows scratch-equivalent baselayers

.gitignore

@@ -13,3 +13,6 @@
 
 # IDEs
 .idea/
+
+# Build timestamp
+tools/bcdhive_generator/.build

Makefile

@@ -1,5 +1,6 @@
 # Go parameters
 GOCMD?=go
+GO_VERSION=$(shell go list -m -f "{{.GoVersion}}")
 PACKAGE_BASE=github.com/buildpacks/imgutil
 
 all: test
@@ -20,5 +21,22 @@ lint: install-golangci-lint
 	@echo "> Linting code..."
 	@golangci-lint run -c golangci.yaml
 
-test: format lint
+tools/bcdhive_generator/.build: $(wildcard tools/bcdhive_generator/*)
+ifneq ($(OS),Windows_NT)
+	@echo "> Building bcdhive-generator in Docker using current golang version"
+	docker build tools/bcdhive_generator --tag bcdhive-generator --build-arg go_version=$(GO_VERSION)
+
+	@touch tools/bcdhive_generator/.build
+else
+	@echo "> Cannot generate on Windows"
+endif
+
+layer/bcdhive_generated.go: layer/windows_baselayer.go tools/bcdhive_generator/.build
+ifneq ($(OS),Windows_NT)
+	$(GOCMD) generate ./...
+else
+	@echo "> Cannot generate on Windows"
+endif
+
+test: layer/bcdhive_generated.go format lint
 	$(GOCMD) test -parallel=1 -count=1 -v ./...

go.sum

@@ -122,14 +122,6 @@ github.com/google/go-cmp v0.2.0 h1:+dTQ8DZQJz0Mb/HjFlkptS1FeQ4cWSnN941F8aEG4SQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
-github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.4.1 h1:/exdXoGamhu5ONeUJH0deniYLWYvQwW66yvlfiiKTu0=
-github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w=
-github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.1 h1:JFrFEBb2xKufg6XkJsJr+WbKb4FQlURi5RUcBveYu9k=
-github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-containerregistry v0.0.0-20200311163244-4b1985e5ea21 h1:rz5VzU1xKHR8HDtifeAJ+SPwE0v3YW0AEien/Lobgww=

layer/windows_baselayer.go

@@ -0,0 +1,84 @@
+package layer
+
+import (
+	"archive/tar"
+	"bytes"
+	"io"
+)
+
+// Generate using `make generate`
+//go:generate docker run --rm -v $PWD:/out/ bcdhive-generator -file=/out/layer/bcdhive_generated.go -package=layer -func=BaseLayerBCD
+
+// Windows base layers must follow this pattern:
+//  \-> UtilityVM/Files/EFI/Microsoft/Boot/BCD   (file must exist and a valid BCD format - from bcdhive_gen)
+//  \-> Files/Windows/System32/config/DEFAULT   (file and must exist but can be empty)
+//  \-> Files/Windows/System32/config/SAM       (file must exist but can be empty)
+//  \-> Files/Windows/System32/config/SECURITY  (file must exist but can be empty)
+//  \-> Files/Windows/System32/config/SOFTWARE  (file must exist but can be empty)
+//  \-> Files/Windows/System32/config/SYSTEM    (file must exist but can be empty)
+// Refs:
+// https://github.com/microsoft/hcsshim/blob/master/internal/wclayer/legacy.go
+// https://github.com/moby/moby/blob/master/daemon/graphdriver/windows/windows.go#L48
+func WindowsBaseLayer() (io.Reader, error) {
+	bcdBytes, err := BaseLayerBCD()
+	if err != nil {
+		return nil, err
+	}
+
+	layerBuffer := &bytes.Buffer{}
+	tw := tar.NewWriter(layerBuffer)
+
+	if err := tw.WriteHeader(&tar.Header{Name: "UtilityVM", Typeflag: tar.TypeDir}); err != nil {
+		return nil, err
+	}
+	if err := tw.WriteHeader(&tar.Header{Name: "UtilityVM/Files", Typeflag: tar.TypeDir}); err != nil {
+		return nil, err
+	}
+	if err := tw.WriteHeader(&tar.Header{Name: "UtilityVM/Files/EFI", Typeflag: tar.TypeDir}); err != nil {
+		return nil, err
+	}
+	if err := tw.WriteHeader(&tar.Header{Name: "UtilityVM/Files/EFI/Microsoft", Typeflag: tar.TypeDir}); err != nil {
+		return nil, err
+	}
+	if err := tw.WriteHeader(&tar.Header{Name: "UtilityVM/Files/EFI/Microsoft/Boot", Typeflag: tar.TypeDir}); err != nil {
+		return nil, err
+	}
+
+	if err := tw.WriteHeader(&tar.Header{Name: "UtilityVM/Files/EFI/Microsoft/Boot/BCD", Size: int64(len(bcdBytes)), Mode: 0644}); err != nil {
+		return nil, err
+	}
+	if _, err := tw.Write(bcdBytes); err != nil {
+		return nil, err
+	}
+
+	if err := tw.WriteHeader(&tar.Header{Name: "Files", Typeflag: tar.TypeDir}); err != nil {
+		return nil, err
+	}
+	if err := tw.WriteHeader(&tar.Header{Name: "Files/Windows", Typeflag: tar.TypeDir}); err != nil {
+		return nil, err
+	}
+	if err := tw.WriteHeader(&tar.Header{Name: "Files/Windows/System32", Typeflag: tar.TypeDir}); err != nil {
+		return nil, err
+	}
+	if err := tw.WriteHeader(&tar.Header{Name: "Files/Windows/System32/config", Typeflag: tar.TypeDir}); err != nil {
+		return nil, err
+	}
+
+	if err := tw.WriteHeader(&tar.Header{Name: "Files/Windows/System32/config/DEFAULT", Size: 0, Mode: 0644}); err != nil {
+		return nil, err
+	}
+	if err := tw.WriteHeader(&tar.Header{Name: "Files/Windows/System32/config/SAM", Size: 0, Mode: 0644}); err != nil {
+		return nil, err
+	}
+	if err := tw.WriteHeader(&tar.Header{Name: "Files/Windows/System32/config/SECURITY", Size: 0, Mode: 0644}); err != nil {
+		return nil, err
+	}
+	if err := tw.WriteHeader(&tar.Header{Name: "Files/Windows/System32/config/SOFTWARE", Size: 0, Mode: 0644}); err != nil {
+		return nil, err
+	}
+	if err := tw.WriteHeader(&tar.Header{Name: "Files/Windows/System32/config/SYSTEM", Size: 0, Mode: 0644}); err != nil {
+		return nil, err
+	}
+
+	return layerBuffer, nil
+}

local/local_test.go

@@ -1080,7 +1080,6 @@ func testImage(t *testing.T, when spec.G, it spec.S) {
 				repoName,
 				dockerClient,
 				local.FromBaseImage(repoName),
-				local.WithPreviousImage(repoName),
 			)
 			h.AssertNil(t, err)
 

testhelpers/testhelpers.go

@@ -2,25 +2,23 @@ package testhelpers
 
 import (
 	"archive/tar"
-	"bytes"
-	"compress/gzip"
 	"context"
 	"crypto/sha256"
-	"encoding/base64"
 	"encoding/hex"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"math/rand"
 	"net/http"
 	"os"
-	"path"
 	"regexp"
 	"strings"
 	"sync"
 	"testing"
 	"time"
 
+	"github.com/buildpacks/imgutil/layer"
+
 	dockertypes "github.com/docker/docker/api/types"
 	dockercli "github.com/docker/docker/client"
 	"github.com/google/go-cmp/cmp"
@@ -222,137 +220,78 @@ func CreateSingleFileTarReader(path, txt string) io.ReadCloser {
 	pr, pw := io.Pipe()
 
 	go func() {
-		var err error
-		defer func() {
+		// Use the regular tar.Writer, as this isn't a layer tar.
+		tw := tar.NewWriter(pw)
+
+		if err := tw.WriteHeader(&tar.Header{Name: path, Size: int64(len(txt)), Mode: 0644}); err != nil {
 			pw.CloseWithError(err)
-		}()
+		}
 
-		tw := tar.NewWriter(pw)
-		defer tw.Close()
+		if _, err := tw.Write([]byte(txt)); err != nil {
+			pw.CloseWithError(err)
+		}
 
-		err = writeTarSingleFileLinux(tw, path, txt) // Use the Linux writer, as this isn't a layer tar.
+		if err := tw.Close(); err != nil {
+			pw.CloseWithError(err)
+		}
+
+		if err := pw.Close(); err != nil {
+			pw.CloseWithError(err)
+		}
 	}()
 
 	return pr
 }
 
-func CreateSingleFileLayerTar(layerPath, txt, osType string) (string, error) {
-	tarFile, err := ioutil.TempFile("", "create-single-file-layer-tar-path")
-	if err != nil {
-		return "", err
-	}
-	defer tarFile.Close()
-
-	tw := tar.NewWriter(tarFile)
-	defer tw.Close()
+type layerWriter interface {
+	WriteHeader(*tar.Header) error
+	Write([]byte) (int, error)
+	Close() error
+}
 
-	// regular Linux layer
-	writeFunc := writeTarSingleFileLinux
+func getLayerWriter(osType string, file *os.File) layerWriter {
 	if osType == "windows" {
-		// regular Windows layer
-		writeFunc = writeTarSingleFileWindows
+		return layer.NewWindowsWriter(file)
 	}
+	return tar.NewWriter(file)
+}
 
-	err = writeFunc(tw, layerPath, txt)
+func CreateSingleFileLayerTar(layerPath, txt, osType string) (string, error) {
+	tarFile, err := ioutil.TempFile("", "create-single-file-layer-tar-path")
 	if err != nil {
 		return "", err
 	}
+	defer tarFile.Close()
 
-	return tarFile.Name(), nil
-}
+	tw := getLayerWriter(osType, tarFile)
 
-func writeTarSingleFileLinux(tw *tar.Writer, layerPath, txt string) error {
 	if err := tw.WriteHeader(&tar.Header{Name: layerPath, Size: int64(len(txt)), Mode: 0644}); err != nil {
-		return err
+		return "", err
 	}
 
 	if _, err := tw.Write([]byte(txt)); err != nil {
-		return err
+		return "", err
+	}
+
+	if err := tw.Close(); err != nil {
+		return "", err
 	}
 
-	return nil
+	return tarFile.Name(), nil
 }
 
-// WindowsBaseLayer returns a minimal windows base layer.
-// This base layer cannot use for running but can be used for saving to a Windows daemon and container creation.
-// Windows image layers must follow this pattern¹:
-// - base layer² (always required; tar file with relative paths without "/" prefix; all parent directories require own tar entries)
-//   \-> Files/Windows/System32/config/DEFAULT   (file must exist but can be empty)
-//   \-> Files/Windows/System32/config/SAM       (file must exist but can be empty)
-//   \-> Files/Windows/System32/config/SECURITY  (file must exist but can be empty)
-//   \-> Files/Windows/System32/config/SOFTWARE  (file must exist but can be empty)
-//   \-> Files/Windows/System32/config/SYSTEM    (file must exist but can be empty)
-//   \-> UtilityVM/Files/EFI/Microsoft/Boot/BCD   (file must exist and a valid BCD format - via `bcdedit` tool as below)
-// - normal or top layer (optional; tar file with relative paths without "/" prefix; all parent directories require own tar entries)
-//   \-> Files/                   (required directory entry)
-//   \-> Files/mystuff.exe        (optional container filesystem files - C:\mystuff.exe)
-//   \-> Hives/                   (required directory entry)
-//   \-> Hives/DefaultUser_Delta  (optional Windows reg hive delta; BCD format - HKEY_USERS\.DEFAULT additional content)
-//   \-> Hives/Sam_Delta          (optional Windows reg hive delta; BCD format - HKEY_LOCAL_MACHINE\SAM additional content)
-//   \-> Hives/Security_Delta     (optional Windows reg hive delta; BCD format - HKEY_LOCAL_MACHINE\SECURITY additional content)
-//   \-> Hives/Software_Delta     (optional Windows reg hive delta; BCD format - HKEY_LOCAL_MACHINE\SOFTWARE additional content)
-//   \-> Hives/System_Delta       (optional Windows reg hive delta; BCD format - HKEY_LOCAL_MACHINE\SYSTEM additional content)
-// 1. This was all discovered experimentally and should be considered an undocumented API, subject to change when the Windows Daemon internals change
-// 2. There are many other files in an "real" base layer but this is the minimum set which a Daemon can store and use to create an container
 func WindowsBaseLayer(t *testing.T) string {
 	tarFile, err := ioutil.TempFile("", "windows-base-layer.tar")
 	AssertNil(t, err)
+	defer tarFile.Close()
 
-	tw := tar.NewWriter(tarFile)
-
-	//Valid BCD file required, containing Windows Boot Manager and Windows Boot Loader sections
-	//Note: Gzip/Base64 encoded only to inline the binary BCD file here
-	//CMD: `bcdedit /createstore c:\output-bcd & bcdedit /create {6a6c1f1b-59d4-11ea-9438-9402e6abd998} /d buildpacks.io /application osloader /store c:\output-bcd & bcdedit /create {bootmgr} /store c:\output-bcd & bcdedit /set {bootmgr} default {6a6c1f1b-59d4-11ea-9438-9402e6abd998} /store c:\output-bcd & bcdedit /enum all /store c:\output-bcd`
-	//BASH: `gzip --stdout --best output-bcd | base64`
-	bcdGzipBase64 := "H4sIABeDWF4CA+1YTWgTQRR+m2zSFItGkaJQcKXgyZX8bNKklxatUvyJoh4qKjS7O7GxzQ9Jai210JtFQXrUW4/e2ostCF4EoSCCF6HHQi9FWsxJepH43s5us02XYlFBcL7l7U7evHnzzZtvoJ0Ke5A7BABkoU/f3qxvfZEkbPuBg9oKNcK8fQ/68LkHBvTiuwTjUIOy9VZBR68J++NZ/sXU/J2vR99d/thxdzOz0PqbYp63+CqFWuH7wg+LGwj8MfRuvnovqiAgICAgICAgICAgIPB/YETPF8H+/96Bcw9A7flGo1EcPQvrRVginw99Q6cAfHbsEDYwpEFt+s7Y306PuTrQMmziVq1UYTdLpRr5HmNsdRSg38+N8o5Z9w7yzCDlt8ceB+rT7HlPQB8cAYn/CND9hOLjUZaf3xIEjlGelhiJX2wE7gOfy7lQeG2tU4Gt7vZlZ50KNNd5I7B7ncSVvlc91tmGdl1/yIxaFbYxph4EmLXzq/2nd/KHpGZ+RfbO71XHM2hTyWzSiOaiuppIm5oajbKsmtbiKXxFYiyZ1c10OjUNUMccZZxkGDkMsKpBfBS/s68KPHlbX3Kv10HDBvnXkKezr06/Yu0CB90dUe5KvlzLl7icNjB2LOeDbYn30VqpJmvofzTaZo2d8/H6k11hk5lsgQH1n4cLMACRlofDi/eItJc3ueoS15SbdwhN3od33eItQQqDorFIhPOVacyMH5SwbPO9PVlmgy79Un3I2n9Rvych+Ff0+6Grc9ldF6c/7PfWV9hDX1Sji2OswIq1qrOPiz5eqxU/7/Oab8XvvQ+f5b37cBityzUf1RqhOfqgvkW5qQ+bj6UPHcYhj1U2oQxZMGAUqnAOPSWMI33Pyc3z5j7P7vM2GDzgeUubLJtKxgw1YZimqrGeiJo1jKiai8f0uKaZWk86Md3UPdWezuiqzMd66XZV9q7XRuDgunXr1AfhXToFuy4rgaZOup82dvFwdLIW/D2djAQ4t3qA961avMIWL8leA30v5SuFiWyFXSuZ+VyemV68KIdXfYYlbz1lXLxicUtPcec8zwa5z9EXxYbb9uqLeExBEnWVRGVFIYemgwoJSKPeNGxF8WHYr6JHgzik7FYEYuinkTpGpvFJwfQO/5ch8beGgICAgICAwL+BnwAgqcMAIAAA"
-	bcdGzip, _ := base64.StdEncoding.DecodeString(bcdGzipBase64)
-	bcdReader, _ := gzip.NewReader(bytes.NewBuffer(bcdGzip))
-	bcdBytes, _ := ioutil.ReadAll(bcdReader)
-
-	AssertNil(t, tw.WriteHeader(&tar.Header{Name: "Files", Typeflag: tar.TypeDir}))
-	AssertNil(t, tw.WriteHeader(&tar.Header{Name: "Files/Windows", Typeflag: tar.TypeDir}))
-	AssertNil(t, tw.WriteHeader(&tar.Header{Name: "Files/Windows/System32", Typeflag: tar.TypeDir}))
-	AssertNil(t, tw.WriteHeader(&tar.Header{Name: "Files/Windows/System32/config", Typeflag: tar.TypeDir}))
-
-	AssertNil(t, tw.WriteHeader(&tar.Header{Name: "UtilityVM", Typeflag: tar.TypeDir}))
-	AssertNil(t, tw.WriteHeader(&tar.Header{Name: "UtilityVM/Files", Typeflag: tar.TypeDir}))
-	AssertNil(t, tw.WriteHeader(&tar.Header{Name: "UtilityVM/Files/EFI", Typeflag: tar.TypeDir}))
-	AssertNil(t, tw.WriteHeader(&tar.Header{Name: "UtilityVM/Files/EFI/Microsoft", Typeflag: tar.TypeDir}))
-	AssertNil(t, tw.WriteHeader(&tar.Header{Name: "UtilityVM/Files/EFI/Microsoft/Boot", Typeflag: tar.TypeDir}))
-
-	AssertNil(t, tw.WriteHeader(&tar.Header{Name: "Files/Windows/System32/config/DEFAULT", Size: 0, Mode: 0644}))
-	AssertNil(t, tw.WriteHeader(&tar.Header{Name: "Files/Windows/System32/config/SAM", Size: 0, Mode: 0644}))
-	AssertNil(t, tw.WriteHeader(&tar.Header{Name: "Files/Windows/System32/config/SECURITY", Size: 0, Mode: 0644}))
-	AssertNil(t, tw.WriteHeader(&tar.Header{Name: "Files/Windows/System32/config/SOFTWARE", Size: 0, Mode: 0644}))
-	AssertNil(t, tw.WriteHeader(&tar.Header{Name: "Files/Windows/System32/config/SYSTEM", Size: 0, Mode: 0644}))
-
-	AssertNil(t, tw.WriteHeader(&tar.Header{Name: "UtilityVM/Files/EFI/Microsoft/Boot/BCD", Size: int64(len(bcdBytes)), Mode: 0644}))
-	_, err = tw.Write(bcdBytes)
+	baseLayer, err := layer.WindowsBaseLayer()
 	AssertNil(t, err)
 
-	return tarFile.Name()
-}
-
-func writeTarSingleFileWindows(tw *tar.Writer, containerPath, txt string) error {
-	// root Windows layer directories
-	if err := tw.WriteHeader(&tar.Header{Name: "Files", Typeflag: tar.TypeDir}); err != nil {
-		return err
-	}
-	if err := tw.WriteHeader(&tar.Header{Name: "Hives", Typeflag: tar.TypeDir}); err != nil {
-		return err
-	}
-
-	// prepend file entries with "Files"
-	layerPath := path.Join("Files", containerPath)
-	if err := tw.WriteHeader(&tar.Header{Name: layerPath, Size: int64(len(txt)), Mode: 0644}); err != nil {
-		return err
-	}
-
-	if _, err := tw.Write([]byte(txt)); err != nil {
-		return err
-	}
+	_, err = io.Copy(tarFile, baseLayer)
+	AssertNil(t, err)
 
-	return nil
+	return tarFile.Name()
 }
 
 func FetchManifestLayers(t *testing.T, repoName string) []string {

tools/bcdhive_generator/Dockerfile

@@ -0,0 +1,15 @@
+ARG go_version
+FROM golang:${go_version}-buster
+
+RUN apt update \
+    && apt install -y libhivex-dev libhivex-bin libwin-hivex-perl
+
+COPY . /src
+
+WORKDIR /src
+
+RUN go generate ./ \
+    && go test -parallel=1 -count=1 -v . \
+    && go install .
+
+ENTRYPOINT ["/go/bin/bcdhive_gen"]