mantle/kola/testiso: support testing coreos.liveiso.fromram installs

mantle/cmd/kola/qemuexec.go

@@ -125,6 +125,41 @@ func parseBindOpt(s string) (string, string, error) {
 	return parts[0], parts[1], nil
 }
 
+// buildDiskFromOptions generates a disk image template using the process-global
+// defaults that were parsed from command line arguments.
+func buildDiskFromOptions() *platform.Disk {
+	channel := "virtio"
+	if kola.QEMUOptions.Nvme {
+		channel = "nvme"
+	}
+	sectorSize := 0
+	if kola.QEMUOptions.Native4k {
+		sectorSize = 4096
+	}
+	options := []string{}
+	if kola.QEMUOptions.DriveOpts != "" {
+		options = append(options, strings.Split(kola.QEMUOptions.DriveOpts, ",")...)
+	}
+	// If there was no disk image specified and no size then just
+	// default to an arbitrary value of 12G for the blank disk image.
+	size := kola.QEMUOptions.DiskSize
+	if kola.QEMUOptions.DiskImage == "" && kola.QEMUOptions.DiskSize == "" {
+		size = "12G"
+	}
+	// Build the disk definition. Note that if kola.QEMUOptions.DiskImage is
+	// "" we'll just end up with a blank disk image, which is what we want.
+	disk := &platform.Disk{
+		BackingFile:   kola.QEMUOptions.DiskImage,
+		Channel:       channel,
+		Size:          size,
+		SectorSize:    sectorSize,
+		DriveOpts:     options,
+		MultiPathDisk: kola.QEMUOptions.MultiPathDisk,
+		NbdDisk:       kola.QEMUOptions.NbdDisk,
+	}
+	return disk
+}
+
 func runQemuExec(cmd *cobra.Command, args []string) error {
 	var err error
 
@@ -283,36 +318,22 @@ func runQemuExec(cmd *cobra.Command, args []string) error {
 		builder.Firmware = kola.QEMUOptions.Firmware
 	}
 	if kola.QEMUOptions.DiskImage != "" {
-		channel := "virtio"
-		if kola.QEMUOptions.Nvme {
-			channel = "nvme"
-		}
-		sectorSize := 0
-		if kola.QEMUOptions.Native4k {
-			sectorSize = 4096
-		}
-		options := []string{}
-		if kola.QEMUOptions.DriveOpts != "" {
-			options = append(options, strings.Split(kola.QEMUOptions.DriveOpts, ",")...)
+		if err := builder.AddBootDisk(buildDiskFromOptions()); err != nil {
+			return err
 		}
-		err = builder.AddBootDisk(&platform.Disk{
-			BackingFile:   kola.QEMUOptions.DiskImage,
-			Channel:       channel,
-			Size:          kola.QEMUOptions.DiskSize,
-			SectorSize:    sectorSize,
-			DriveOpts:     options,
-			MultiPathDisk: kola.QEMUOptions.MultiPathDisk,
-			NbdDisk:       kola.QEMUOptions.NbdDisk,
-		})
 		if err != nil {
 			return err
 		}
 	}
 	if kola.QEMUIsoOptions.IsoPath != "" {
-		err := builder.AddIso(kola.QEMUIsoOptions.IsoPath, "", kola.QEMUIsoOptions.AsDisk)
+		err := builder.AddIso(kola.QEMUIsoOptions.IsoPath, "bootindex=3", kola.QEMUIsoOptions.AsDisk)
 		if err != nil {
 			return err
 		}
+		// Add a blank disk (this is a disk we can install to)
+		if err := builder.AddBootDisk(buildDiskFromOptions()); err != nil {
+			return err
+		}
 	}
 	builder.Hostname = hostname
 	// for historical reasons, both --memory and --qemu-memory are supported

mantle/cmd/kola/testiso.go

@@ -64,6 +64,7 @@ var (
 	enableUefi       bool
 	enableUefiSecure bool
 	isOffline        bool
+	isISOFromRAM     bool
 
 	// The iso-as-disk tests are only supported in x86_64 because other
 	// architectures don't have the required hybrid partition table.
@@ -79,7 +80,7 @@ var (
 		"iso-live-login.4k.uefi",
 		"iso-offline-install.bios",
 		"iso-offline-install.mpath.bios",
-		"iso-offline-install.4k.uefi",
+		"iso-offline-install-fromram.4k.uefi",
 		"miniso-install.bios",
 		"miniso-install.nm.bios",
 		"miniso-install.4k.uefi",
@@ -101,7 +102,7 @@ var (
 	}
 	tests_ppc64le = []string{
 		"iso-live-login.ppcfw",
-		"iso-offline-install.ppcfw",
+		"iso-offline-install-fromram.ppcfw",
 		"iso-offline-install.mpath.ppcfw",
 		"iso-offline-install.4k.ppcfw",
 		"miniso-install.ppcfw",
@@ -114,7 +115,7 @@ var (
 	tests_aarch64 = []string{
 		"iso-live-login.uefi",
 		"iso-live-login.4k.uefi",
-		"iso-offline-install.uefi",
+		"iso-offline-install-fromram.uefi",
 		"iso-offline-install.mpath.uefi",
 		"iso-offline-install.4k.uefi",
 		"miniso-install.uefi",
@@ -130,6 +131,8 @@ var (
 
 const (
 	installTimeout = 10 * time.Minute
+	// https://github.com/coreos/fedora-coreos-config/pull/2544
+	liveISOFromRAMKarg = "coreos.liveiso.fromram"
 )
 
 var liveOKSignal = "live-test-OK"
@@ -220,6 +223,8 @@ ExecStart=/bin/bash -c '[[ $(findmnt -nvro SOURCE /sysroot) == /dev/mapper/mpath
 [Install]
 RequiredBy=multi-user.target`
 
+// This test is broken. Please fix!
+// https://github.com/coreos/coreos-assembler/issues/3554
 var verifyNoEFIBootEntry = `[Unit]
 Description=TestISO Verify No EFI Boot Entry
 OnFailure=emergency.target
@@ -236,6 +241,24 @@ RequiredBy=coreos-installer.target
 # for iso-as-disk
 RequiredBy=multi-user.target`
 
+// Unit to check that /run/media/iso is not mounted when
+// coreos.liveiso.fromram kernel argument is passed
+var isoNotMountedUnit = `[Unit]
+Description=Verify ISO is not mounted when coreos.liveiso.fromram
+OnFailure=emergency.target
+OnFailureJobMode=isolate
+ConditionKernelCommandLine=coreos.liveiso.fromram
+[Service]
+Type=oneshot
+StandardOutput=kmsg+console
+StandardError=kmsg+console
+RemainAfterExit=yes
+# Would like to use SuccessExitStatus but it doesn't support what
+# we want: https://github.com/systemd/systemd/issues/10297#issuecomment-1672002635
+ExecStart=bash -c "if mountpoint /run/media/iso 2>/dev/null; then exit 1; fi"
+[Install]
+RequiredBy=coreos-installer.target`
+
 var nmConnectionId = "CoreOS DHCP"
 var nmConnectionFile = "coreos-dhcp.nmconnection"
 var nmConnection = fmt.Sprintf(`[connection]
@@ -519,6 +542,12 @@ func runTestIso(cmd *cobra.Command, args []string) error {
 		if kola.HasString("offline", strings.Split(components[0], "-")) {
 			isOffline = true
 		}
+		// For fromram it is a part of the first component. i.e. for
+		// iso-offline-install-fromram.uefi we need to search for 'fromram' in
+		// iso-offline-install-fromram, which is currently in components[0].
+		if kola.HasString("fromram", strings.Split(components[0], "-")) {
+			isISOFromRAM = true
+		}
 
 		switch components[0] {
 		case "pxe-offline-install", "pxe-online-install":
@@ -527,7 +556,7 @@ func runTestIso(cmd *cobra.Command, args []string) error {
 			duration, err = testAsDisk(ctx, filepath.Join(outputDir, test))
 		case "iso-live-login":
 			duration, err = testLiveLogin(ctx, filepath.Join(outputDir, test))
-		case "iso-install", "iso-offline-install":
+		case "iso-install", "iso-offline-install", "iso-offline-install-fromram":
 			duration, err = testLiveIso(ctx, inst, filepath.Join(outputDir, test), false)
 		case "miniso-install":
 			duration, err = testLiveIso(ctx, inst, filepath.Join(outputDir, test), true)
@@ -711,13 +740,15 @@ func testLiveIso(ctx context.Context, inst platform.Install, outdir string, mini
 		return 0, err
 	}
 
+	var isoKernelArgs []string
 	var keys []string
 	keys = append(keys, strings.TrimSpace(string(sshPubKeyBuf)))
 	virtioJournalConfig.AddAuthorizedKeys("core", keys)
 
 	liveConfig := *virtioJournalConfig
 	liveConfig.AddSystemdUnit("live-signal-ok.service", liveSignalOKUnit, conf.Enable)
 	liveConfig.AddSystemdUnit("verify-no-efi-boot-entry.service", verifyNoEFIBootEntry, conf.Enable)
+	liveConfig.AddSystemdUnit("iso-not-mounted-when-fromram.service", isoNotMountedUnit, conf.Enable)
 	liveConfig.AddSystemdUnit("coreos-test-entered-emergency-target.service", signalFailureUnit, conf.Enable)
 
 	targetConfig := *virtioJournalConfig
@@ -738,7 +769,19 @@ func testLiveIso(ctx context.Context, inst platform.Install, outdir string, mini
 		liveConfig.AddFile(nmstateConfigFile, nmstateConfig, 0644)
 	}
 
-	mach, err := inst.InstallViaISOEmbed(nil, liveConfig, targetConfig, outdir, isOffline, minimal)
+	if isISOFromRAM {
+		isoKernelArgs = append(isoKernelArgs, liveISOFromRAMKarg)
+	}
+
+	// Sometimes the logs that stream from various virtio streams can be
+	// incomplete because they depend on services inside the guest.
+	// When you are debugging earlyboot/initramfs issues this can be
+	// problematic. Let's add a hook here to enable more debugging.
+	if _, ok := os.LookupEnv("COSA_TESTISO_DEBUG"); ok {
+		isoKernelArgs = append(isoKernelArgs, "systemd.log_color=0 systemd.log_level=debug systemd.log_target=console")
+	}
+
+	mach, err := inst.InstallViaISOEmbed(isoKernelArgs, liveConfig, targetConfig, outdir, isOffline, minimal)
 	if err != nil {
 		return 0, errors.Wrapf(err, "running iso install")
 	}

mantle/platform/metal.go

@@ -599,11 +599,6 @@ func (inst *Install) InstallViaISOEmbed(kargs []string, liveIgnition, targetIgni
 		return nil, fmt.Errorf("Cannot use `--add-nm-keyfile` with offline mode")
 	}
 
-	// XXX: we do support this now, via `coreos-installer iso kargs`
-	if len(inst.kargs) > 0 {
-		return nil, errors.New("injecting kargs is not supported yet, see https://github.com/coreos/coreos-installer/issues/164")
-	}
-
 	installerConfig := installerConfig{
 		IgnitionFile: "/var/opt/pointer.ign",
 		DestDevice:   "/dev/vda",
@@ -645,6 +640,20 @@ func (inst *Install) InstallViaISOEmbed(kargs []string, liveIgnition, targetIgni
 
 	builddir := inst.CosaBuild.Dir
 	srcisopath := filepath.Join(builddir, inst.CosaBuild.Meta.BuildArtifacts.LiveIso.Path)
+
+	// Copy the ISO to a new location for modification.
+	// This is a bit awkward; we copy here, but QemuBuilder will also copy
+	// again (in `setupIso()`). I didn't want to lower the NM keyfile stuff
+	// into QemuBuilder. And plus, both tempdirs should be in /var/tmp so
+	// the `cp --reflink=auto` that QemuBuilder does should just reflink.
+	newIso := filepath.Join(tempdir, "install.iso")
+	cmd := exec.Command("cp", "--reflink=auto", srcisopath, newIso)
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		return nil, errors.Wrapf(err, "copying iso")
+	}
+	srcisopath = newIso
+
 	var metalimg string
 	if inst.Native4k {
 		metalimg = inst.CosaBuild.Meta.BuildArtifacts.Metal4KNative.Path
@@ -732,33 +741,31 @@ func (inst *Install) InstallViaISOEmbed(kargs []string, liveIgnition, targetIgni
 		keyfileArgs = append(keyfileArgs, "--keyfile", path)
 	}
 	if len(keyfileArgs) > 0 {
-		// This is a bit awkward; we copy here, but QemuBuilder will also copy
-		// again (in `setupIso()`). I didn't want to lower the NM keyfile stuff
-		// into QemuBuilder. And plus, both tempdirs should be in /var/tmp so
-		// the `cp --reflink=auto` that QemuBuilder does should just reflink.
-		newIso := filepath.Join(tempdir, "install.iso")
-		cmd := exec.Command("cp", "--reflink=auto", srcisopath, newIso)
-		cmd.Stderr = os.Stderr
-		if err := cmd.Run(); err != nil {
-			return nil, errors.Wrapf(err, "copying iso")
-		}
 
-		args := []string{"iso", "network", "embed", newIso}
+		args := []string{"iso", "network", "embed", srcisopath}
 		args = append(args, keyfileArgs...)
 		cmd = exec.Command("coreos-installer", args...)
 		cmd.Stderr = os.Stderr
 		if err := cmd.Run(); err != nil {
 			return nil, errors.Wrapf(err, "running coreos-installer iso network embed")
 		}
 
+		installerConfig.CopyNetwork = true
+
 		// force networking on in the initrd to verify the keyfile was used
-		cmd = exec.Command("coreos-installer", "iso", "kargs", "modify", newIso, "--append", "rd.neednet=1")
+		inst.kargs = append(inst.kargs, "rd.neednet=1")
+	}
+
+	if len(inst.kargs) > 0 {
+		args := []string{"iso", "kargs", "modify", srcisopath}
+		for _, karg := range inst.kargs {
+			args = append(args, "--append", karg)
+		}
+		cmd = exec.Command("coreos-installer", args...)
 		cmd.Stderr = os.Stderr
 		if err := cmd.Run(); err != nil {
-			return nil, errors.Wrapf(err, "running coreos-installer iso kargs modify")
+			return nil, errors.Wrapf(err, "running coreos-installer iso kargs")
 		}
-		srcisopath = newIso
-		installerConfig.CopyNetwork = true
 	}
 
 	if inst.Insecure {

mantle/platform/qemu.go

@@ -1077,18 +1077,22 @@ func (builder *QemuBuilder) addDiskImpl(disk *Disk, primary bool) error {
 		return err
 	}
 	if primary {
-		// If the board doesn't support -fw_cfg or we were explicitly
-		// requested, inject via libguestfs on the primary disk.
-		if err := builder.renderIgnition(); err != nil {
-			return errors.Wrapf(err, "rendering ignition")
-		}
-		requiresInjection := builder.ConfigFile != "" && builder.ForceConfigInjection
-		if requiresInjection || builder.AppendFirstbootKernelArgs != "" || builder.AppendKernelArgs != "" {
-			if err := setupPreboot(builder.architecture, builder.ConfigFile, builder.AppendFirstbootKernelArgs, builder.AppendKernelArgs,
-				disk.dstFileName, disk.SectorSize); err != nil {
-				return errors.Wrapf(err, "ignition injection with guestfs failed")
+		// Only try to inject config if it hasn't already been injected somewhere
+		// else, which can happen when running an ISO install.
+		if !builder.configInjected {
+			// If the board doesn't support -fw_cfg or we were explicitly
+			// requested, inject via libguestfs on the primary disk.
+			if err := builder.renderIgnition(); err != nil {
+				return errors.Wrapf(err, "rendering ignition")
+			}
+			requiresInjection := builder.ConfigFile != "" && builder.ForceConfigInjection
+			if requiresInjection || builder.AppendFirstbootKernelArgs != "" || builder.AppendKernelArgs != "" {
+				if err := setupPreboot(builder.architecture, builder.ConfigFile, builder.AppendFirstbootKernelArgs, builder.AppendKernelArgs,
+					disk.dstFileName, disk.SectorSize); err != nil {
+					return errors.Wrapf(err, "ignition injection with guestfs failed")
+				}
+				builder.configInjected = true
 			}
-			builder.configInjected = true
 		}
 	}
 	diskOpts := disk.DeviceOpts