⚠️ add mbr disk generation (#45)

* add mbr disk generation Signed-off-by: mudler <[email protected]> * Specify iso Signed-off-by: mudler <[email protected]> * tests: use a default config Signed-off-by: mudler <[email protected]> * debug Signed-off-by: mudler <[email protected]> * try fix tests Signed-off-by: mudler <[email protected]> * make space for the worker Signed-off-by: mudler <[email protected]> * ci: timeout Signed-off-by: mudler <[email protected]> * debug Signed-off-by: mudler <[email protected]> * test Signed-off-by: mudler <[email protected]> * try to change image Signed-off-by: mudler <[email protected]> * write config also on the other set of tests Signed-off-by: mudler <[email protected]> * wire up more config, better defaults Signed-off-by: mudler <[email protected]> * try self-hosted runners Signed-off-by: mudler <[email protected]> * Do not run in earthly Signed-off-by: mudler <[email protected]> * replace 'iso' build dir to 'build' More meaningful as we don't build only ISOs Signed-off-by: mudler <[email protected]> * simplify test Signed-off-by: mudler <[email protected]> * speedup, try to enable kvm Signed-off-by: mudler <[email protected]> * fixup test output check Signed-off-by: mudler <[email protected]> --------- Signed-off-by: mudler <[email protected]>
kairos-io · Oct 19, 2023 · 8d13cc9 · 8d13cc9
1 parent c6614fc
commit 8d13cc9
Show file tree

Hide file tree

Showing 11 changed files with 218 additions and 31 deletions.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -9,12 +9,17 @@ concurrency:
  cancel-in-progress: true
 jobs:
  tests:
- runs-on: ubuntu-latest
+ runs-on: kvm
  steps:
  - name: Checkout code
  uses: actions/checkout@v2
  with:
  fetch-depth: 0
+ - name: Set up Go
+ uses: actions/setup-go@v4
+ with:
+ go-version: 1.21
  - name: Run tests
  run: |
- docker run --privileged -v /var/run/docker.sock:/var/run/docker.sock --rm -t -v $(pwd):/workspace -v earthly-tmp:/tmp/earthly:rw earthly/earthly:v0.6.21 --allow-privileged +test
+ docker build -t auroraboot:latest .
+ sudo go run github.com/onsi/ginkgo/v2/ginkgo -r -p --fail-fast --timeout=2h ./...
diff --git a/Dockerfile b/Dockerfile
@@ -1,9 +1,11 @@
+ARG VERSION=v0.9.0
+
 FROM golang as builder
 ADD . /work
 RUN cd /work && \
  CGO_ENABLED=0 go build -o auroraboot
 
-FROM quay.io/kairos/osbuilder-tools
+FROM quay.io/kairos/osbuilder-tools:$VERSION
 COPY --from=builder /work/auroraboot /usr/bin/auroraboot
-
+RUN zypper in -y qemu
 ENTRYPOINT ["/usr/bin/auroraboot"]
diff --git a/Earthfile b/Earthfile
@@ -1,16 +1,33 @@
 VERSION 0.6
+ARG OSBUILDER_VERSION=v0.9.0
 
 image:
- FROM DOCKERFILE -f Dockerfile .
+ FROM DOCKERFILE --build-arg VERSION=$OSBUILDER_VERSION -f Dockerfile .
+ RUN zypper in -y qemu
+
+test-label:
+ FROM alpine
+ WORKDIR /test
+ RUN apk add go docker jq
+ ENV GOPATH=/go
+ ENV FIXTURE_CONFIG=/test/tests/fixtures/raw_disk.yaml
+ ARG LABEL
+ COPY . .
+ WITH DOCKER \
+ --allow-privileged \
+ --load auroraboot:latest=+image
+ RUN pwd && ls -liah && go install -mod=mod github.com/onsi/ginkgo/v2/ginkgo && /go/bin/ginkgo -r -p --randomize-all --procs 2 --fail-fast --timeout=2h --label-filter="$LABEL" --flake-attempts 3 ./...
+ END
 
 test:
  FROM alpine
  WORKDIR /test
  RUN apk add go docker jq
  ENV GOPATH=/go
+ ENV FIXTURE_CONFIG=/test/tests/fixtures/raw_disk.yaml
  COPY . .
  WITH DOCKER \
  --allow-privileged \
  --load auroraboot:latest=+image
- RUN go install -mod=mod github.com/onsi/ginkgo/v2/ginkgo && /go/bin/ginkgo -r -p --randomize-all --procs 2 --fail-fast --flake-attempts 3 ./...
+ RUN pwd && ls -liah && go install -mod=mod github.com/onsi/ginkgo/v2/ginkgo && /go/bin/ginkgo -r -p --randomize-all --procs 2 --fail-fast --timeout=2h --flake-attempts 3 ./...
  END
diff --git a/deployer/register.go b/deployer/register.go
@@ -32,7 +32,9 @@ const (
  opPreparetmproot = "prepare-temp"
  opExtractNetboot = "extract-netboot"
 
- opGenRawDisk = "gen-raw-disk"
+ opGenRawDisk = "gen-raw-disk"
+ opGenMBRRawDisk = "gen-raw-mbr-disk"
+
  opExtractSquashFS = "extract-squashfs"
 
  opConvertGCE = "convert-gce"
@@ -47,7 +49,7 @@ const (
 
 // Register register the op dag based on the configuration and the artifact wanted.
 func Register(g *herd.Graph, artifact schema.ReleaseArtifact, c schema.Config, cloudConfigFile string) {
- dst := c.StateDir("iso")
+ dst := c.StateDir("build")
  dstNetboot := c.StateDir("netboot")
 
  listenAddr := ":8080"
@@ -75,7 +77,7 @@ func Register(g *herd.Graph, artifact schema.ReleaseArtifact, c schema.Config, c
  isoOption := func() bool { return !fromImage }
  netbootOption := func() bool { return !c.DisableNetboot }
  netbootReleaseOption := func() bool { return !c.DisableNetboot && !fromImage }
- diskIsSet := c.Disk.VHD || c.Disk.RAW || c.Disk.GCE
+ rawDiskIsSet := c.Disk.VHD || c.Disk.RAW || c.Disk.GCE
 
  // Pull locak docker daemon if container image starts with docker://
  containerImage := artifact.ContainerImage
@@ -115,7 +117,7 @@ func Register(g *herd.Graph, artifact schema.ReleaseArtifact, c schema.Config, c
  herd.EnableIf(fromImageOption),
  herd.WithDeps(opPreparetmproot), herd.WithCallback(ops.PullContainerImage(containerImage, tmpRootfs, local)))
  g.Add(opGenISO,
- herd.EnableIf(func() bool { return fromImage && !diskIsSet && c.Disk.ARM == nil }),
+ herd.EnableIf(func() bool { return fromImage && !rawDiskIsSet && c.Disk.ARM == nil }),
  herd.WithDeps(opContainerPull, opCopyCloudConfig), herd.WithCallback(ops.GenISO(kairosDefaultArtifactName, tmpRootfs, dst, c.ISO)))
  g.Add(opExtractNetboot,
  herd.EnableIf(func() bool { return fromImage && !c.DisableNetboot }),
@@ -131,7 +133,7 @@ func Register(g *herd.Graph, artifact schema.ReleaseArtifact, c schema.Config, c
  herd.WithDeps(opPrepareNetboot), herd.WithCallback(ops.DownloadArtifact(artifact.KernelURL(), kernelFile)))
  g.Add(opDownloadSquashFS,
  herd.EnableIf(func() bool {
- return !c.DisableNetboot && !fromImage || diskIsSet && !fromImage || !fromImage && c.Disk.ARM != nil
+ return !c.DisableNetboot && !fromImage || rawDiskIsSet && !fromImage || !fromImage && c.Disk.ARM != nil
  }),
  herd.WithDeps(opPrepareNetboot), herd.WithCallback(ops.DownloadArtifact(artifact.SquashFSURL(), squashFSfile)))
  g.Add(opDownloadISO,
@@ -142,15 +144,28 @@ func Register(g *herd.Graph, artifact schema.ReleaseArtifact, c schema.Config, c
 
  // Extract SquashFS from released asset to build the raw disk image if needed
  g.Add(opExtractSquashFS,
- herd.EnableIf(func() bool { return diskIsSet && !fromImage }),
+ herd.EnableIf(func() bool { return rawDiskIsSet && !fromImage }),
  herd.WithDeps(opDownloadSquashFS), herd.WithCallback(ops.ExtractSquashFS(squashFSfile, tmpRootfs)))
 
  imageOrSquashFS := herd.IfElse(fromImage, herd.WithDeps(opContainerPull), herd.WithDeps(opExtractSquashFS))
 
  g.Add(opGenRawDisk,
- herd.EnableIf(func() bool { return diskIsSet && c.Disk.ARM == nil }),
+ herd.EnableIf(func() bool { return rawDiskIsSet && c.Disk.ARM == nil && !c.Disk.MBR }),
  imageOrSquashFS,
- herd.WithCallback(ops.GenRawDisk(tmpRootfs, filepath.Join(dst, "disk.raw"))))
+ herd.WithCallback(ops.GenEFIRawDisk(tmpRootfs, filepath.Join(dst, "disk.raw"))))
+
+ g.Add(opGenMBRRawDisk,
+ herd.EnableIf(func() bool { return c.Disk.ARM == nil && c.Disk.MBR }),
+ herd.IfElse(isoOption(),
+ herd.WithDeps(opDownloadISO), herd.WithDeps(opGenISO),
+ ),
+ herd.IfElse(isoOption(),
+ herd.WithCallback(
+ ops.GenBIOSRawDisk(c, isoFile, filepath.Join(dst, "disk.raw"))),
+ herd.WithCallback(
+ ops.GenBIOSRawDisk(c, isoFile, filepath.Join(dst, "disk.raw"))),
+ ),
+ )
 
  g.Add(opConvertGCE,
  herd.EnableIf(func() bool { return c.Disk.GCE }),

diff --git a/examples/airgap/build.sh b/examples/airgap/build.sh
@@ -27,4 +27,4 @@ docker run -v $PWD/config.yaml:/config.yaml \
  --set "flavor=fedora" \
  --set "repository=kairos-io/provider-kairos"
 
-echo "Custom ISO ready at $PWD/build/iso/kairos.iso.custom.iso"
+echo "Custom ISO ready at $PWD/build/build/kairos.iso.custom.iso"
diff --git a/examples/airgap/build_docker.sh b/examples/airgap/build_docker.sh
@@ -27,4 +27,4 @@ docker run -v $PWD/config.yaml:/config.yaml \
  --cloud-config /config.yaml \
  --set "state_dir=/tmp/auroraboot"
 
-echo "Custom ISO ready at $PWD/build/iso/kairos.iso"
+echo "Custom ISO ready at $PWD/build/build/kairos.iso"
diff --git a/pkg/ops/disks.go b/pkg/ops/disks.go
@@ -117,7 +117,68 @@ func GenArmDisk(src, dst string, do schema.Config) func(ctx context.Context) err
  }
 }
 
-func GenRawDisk(src, dst string) func(ctx context.Context) error {
+func GenBIOSRawDisk(config schema.Config, srcISO, dst string) func(ctx context.Context) error {
+ cloudConfigFile := filepath.Join(filepath.Dir(dst), "config.yaml")
+ return func(ctx context.Context) error {
+
+ ram := "8096"
+ if config.System.Memory != "" {
+ ram = config.System.Memory
+ }
+ cores := "3"
+ if config.System.Cores != "" {
+ cores = config.System.Cores
+ }
+
+ qemuBin := "qemu-system-x86_64"
+ if config.System.Qemubin != "" {
+ qemuBin = config.System.Qemubin
+ }
+
+ tmp, err := os.MkdirTemp("", "gendisk")
+ if err != nil {
+ return err
+ }
+ defer os.RemoveAll(tmp)
+
+ log.Info().Msgf("Generating MBR disk '%s' from '%s'", dst, srcISO)
+
+ extra := ""
+ if config.System.KVM {
+ extra = "-enable-kvm"
+ }
+ out, err := utils.SH(
+ fmt.Sprintf(`mkdir -p build
+pushd build
+touch meta-data
+cp -rfv %s user-data
+
+mkisofs -output ci.iso -volid cidata -joliet -rock user-data meta-data
+truncate -s "+$((20000*1024*1024))" %s
+
+%s -m %s -smp cores=%s \
+ -nographic \
+ -serial mon:stdio \
+ -rtc base=utc,clock=rt \
+ -chardev socket,path=qga.sock,server,nowait,id=qga0 \
+ -device virtio-serial \
+ -device virtserialport,chardev=qga0,name=org.qemu.guest_agent.0 \
+ -drive if=virtio,media=disk,file=%s \
+ -drive format=raw,media=cdrom,readonly=on,file=%s \
+ -drive format=raw,media=cdrom,readonly=on,file=ci.iso \
+ -boot d %s
+ 
+`, cloudConfigFile, dst, qemuBin, ram, cores, dst, srcISO, extra),
+ )
+ log.Printf("Output '%s'", out)
+ if err != nil {
+ log.Error().Msgf("Generating raw disk '%s' from '%s' to '%s' failed with error '%s'", dst, srcISO, extra, err.Error())
+ }
+ return err
+ }
+}
+
+func GenEFIRawDisk(src, dst string) func(ctx context.Context) error {
  return func(ctx context.Context) error {
  tmp, err := os.MkdirTemp("", "gendisk")
  if err != nil {

diff --git a/pkg/schema/config.go b/pkg/schema/config.go
@@ -34,12 +34,22 @@ type Config struct {
  NetBoot NetBoot `yaml:"netboot"`
 
  Disk Disk `yaml:"disk"`
+
+ System System `yaml:"system"`
+}
+
+type System struct {
+ Memory string `yaml:"memory"`
+ Cores string `yaml:"cores"`
+ Qemubin string `yaml:"qemu_bin"`
+ KVM bool `yaml:"kvm"`
 }
 
 type Disk struct {
  RAW bool `yaml:"raw"`
  GCE bool `yaml:"gce"`
  VHD bool `yaml:"vhd"`
+ MBR bool `yaml:"mbr"`
 
  ARM *ARMDiskOptions `yaml:"arm"`
 }

diff --git a/tests/e2e/arm_test.go b/tests/e2e/arm_test.go
@@ -42,7 +42,7 @@ var _ = Describe("ARM image generation", Label("arm"), func() {
  Expect(out).To(ContainSubstring("done"), out)
  Expect(out).To(ContainSubstring("build-arm-image"), out)
  Expect(err).ToNot(HaveOccurred())
- _, err = os.Stat(filepath.Join(tempDir, "build/iso/disk.img"))
+ _, err = os.Stat(filepath.Join(tempDir, "build/build/disk.img"))
  Expect(err).ToNot(HaveOccurred())
  })
 
@@ -62,7 +62,7 @@ var _ = Describe("ARM image generation", Label("arm"), func() {
  Expect(out).ToNot(ContainSubstring("build-arm-image"), out)
  Expect(out).To(ContainSubstring("prepare_arm"), out)
  Expect(err).ToNot(HaveOccurred())
- _, err = os.Stat(filepath.Join(tempDir, "build/iso/efi.img"))
+ _, err = os.Stat(filepath.Join(tempDir, "build/build/efi.img"))
  Expect(err).ToNot(HaveOccurred())
  })
  })