From 5cf1cfdc7a9e32afa0258214f49dd9299a743590 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 15 Nov 2022 11:37:40 -0500 Subject: [PATCH] qemu: Add support for full emulation With this, one can e.g.: `cosa run --qemu-image fedora-coreos-36*s390x.qcow2 --arch s390x` This was surprisingly easy. It's tempting to do *some* full emulation testing for s390x and ppc64le on x86_64/aarch64. My immediate motivation however is setting up an environment to debug https://github.com/coreos/rpm-ostree/issues/4146 --- mantle/cmd/kola/options.go | 3 + mantle/cmd/kola/qemuexec.go | 9 ++ mantle/platform/machine/unprivqemu/cluster.go | 5 ++ mantle/platform/machine/unprivqemu/flight.go | 1 + mantle/platform/qemu.go | 86 ++++++++++++------- src/deps.txt | 2 + 6 files changed, 74 insertions(+), 32 deletions(-) diff --git a/mantle/cmd/kola/options.go b/mantle/cmd/kola/options.go index e6b5e123eb..0189059d0e 100644 --- a/mantle/cmd/kola/options.go +++ b/mantle/cmd/kola/options.go @@ -298,6 +298,9 @@ func syncOptionsImpl(useCosa bool) error { return err } } + // Currently the `--arch` option is defined in terms of coreos-assembler, but + // we also unconditionally use it for qemu if present. + kola.QEMUOptions.Arch = kola.Options.CosaBuildArch units, _ := root.PersistentFlags().GetStringSlice("debug-systemd-units") for _, unit := range units { diff --git a/mantle/cmd/kola/qemuexec.go b/mantle/cmd/kola/qemuexec.go index 1f20770df6..266969ab42 100644 --- a/mantle/cmd/kola/qemuexec.go +++ b/mantle/cmd/kola/qemuexec.go @@ -47,6 +47,8 @@ var ( usernet bool cpuCountHost bool + architecture string + hostname string ignition string butane string @@ -81,6 +83,7 @@ func init() { cmdQemuExec.Flags().StringSliceVar(&ignitionFragments, "add-ignition", nil, "Append well-known Ignition fragment: [\"autologin\", \"autoresize\"]") cmdQemuExec.Flags().StringVarP(&hostname, "hostname", "", "", "Set hostname via DHCP") cmdQemuExec.Flags().IntVarP(&memory, "memory", "m", 0, "Memory in MB") + cmdQemuExec.Flags().StringVar(&architecture, "arch", "", "Use full emulation for target architecture (e.g. aarch64, x86_64, s390x, ppc64le)") cmdQemuExec.Flags().StringArrayVarP(&addDisks, "add-disk", "D", []string{}, "Additional disk, human readable size (repeatable)") cmdQemuExec.Flags().BoolVar(&cpuCountHost, "auto-cpus", false, "Automatically set number of cpus to host count") cmdQemuExec.Flags().BoolVar(&directIgnition, "ignition-direct", false, "Do not parse Ignition, pass directly to instance") @@ -205,6 +208,12 @@ func runQemuExec(cmd *cobra.Command, args []string) error { builder := platform.NewQemuBuilder() defer builder.Close() + if architecture != "" { + if err := builder.SetArchitecture(architecture); err != nil { + return err + } + } + var config *conf.Conf if butane != "" { buf, err := ioutil.ReadFile(butane) diff --git a/mantle/platform/machine/unprivqemu/cluster.go b/mantle/platform/machine/unprivqemu/cluster.go index 3acd6d3f28..26e3bb4067 100644 --- a/mantle/platform/machine/unprivqemu/cluster.go +++ b/mantle/platform/machine/unprivqemu/cluster.go @@ -101,6 +101,11 @@ func (qc *Cluster) NewMachineWithQemuOptions(userdata *conf.UserData, options pl builder.ConfigFile = confPath defer builder.Close() builder.UUID = qm.id + if qc.flight.opts.Arch != "" { + if err := builder.SetArchitecture(qc.flight.opts.Arch); err != nil { + return nil, err + } + } builder.Firmware = qc.flight.opts.Firmware builder.Swtpm = qc.flight.opts.Swtpm builder.Hostname = fmt.Sprintf("qemu%d", qc.BaseCluster.AllocateMachineSerial()) diff --git a/mantle/platform/machine/unprivqemu/flight.go b/mantle/platform/machine/unprivqemu/flight.go index 6e97732926..f8fc4f1e82 100644 --- a/mantle/platform/machine/unprivqemu/flight.go +++ b/mantle/platform/machine/unprivqemu/flight.go @@ -34,6 +34,7 @@ type Options struct { Board string Firmware string Memory string + Arch string NbdDisk bool MultiPathDisk bool diff --git a/mantle/platform/qemu.go b/mantle/platform/qemu.go index 86dccf452a..494abfeb54 100644 --- a/mantle/platform/qemu.go +++ b/mantle/platform/qemu.go @@ -136,6 +136,7 @@ type bootIso struct { // QemuInstance holds an instantiated VM through its lifecycle. type QemuInstance struct { qemu exec.Cmd + architecture string tempdir string swtpm exec.Cmd nbdServers []exec.Cmd @@ -295,7 +296,10 @@ func (inst *QemuInstance) Destroy() { // is used to boot from the network device (boot once is not supported). For s390x, the boot ordering was not a problem as it // would always read from disk first. For aarch64, the bootindex needs to be switched to boot from disk before a reboot func (inst *QemuInstance) SwitchBootOrder() (err2 error) { - if coreosarch.CurrentRpmArch() != "s390x" && coreosarch.CurrentRpmArch() != "aarch64" { + switch inst.architecture { + case "s390x", "aarch64": + break + default: //Not applicable for other arches return nil } @@ -401,6 +405,8 @@ type QemuBuilder struct { // File to which to redirect the serial console ConsoleFile string + // If set, use QEMU full emulation for the target architecture + architecture string // Memory defaults to 1024 on most architectures, others it may be 2048 Memory int // Processors < 0 means to use host count, unset means 1, values > 1 are directly used @@ -454,10 +460,11 @@ type QemuBuilder struct { // NewQemuBuilder creates a new build for QEMU with default settings. func NewQemuBuilder() *QemuBuilder { ret := QemuBuilder{ - Firmware: "bios", - Swtpm: true, - Pdeathsig: true, - Argv: []string{}, + Firmware: "bios", + Swtpm: true, + Pdeathsig: true, + Argv: []string{}, + architecture: coreosarch.CurrentRpmArch(), } return &ret } @@ -526,15 +533,15 @@ func (builder *QemuBuilder) AddFd(fd *os.File) string { } // virtio returns a virtio device argument for qemu, which is architecture dependent -func virtio(device, args string) string { +func virtio(arch, device, args string) string { var suffix string - switch coreosarch.CurrentRpmArch() { + switch arch { case "x86_64", "ppc64le", "aarch64": suffix = "pci" case "s390x": suffix = "ccw" default: - panic(fmt.Sprintf("RpmArch %s unhandled in virtio()", coreosarch.CurrentRpmArch())) + panic(fmt.Sprintf("RpmArch %s unhandled in virtio()", arch)) } return fmt.Sprintf("virtio-%s-%s,%s", device, suffix, args) } @@ -574,7 +581,7 @@ func (builder *QemuBuilder) setupNetworking() error { netdev += ",restrict=on" } - builder.Append("-netdev", netdev, "-device", virtio("net", "netdev=eth0")) + builder.Append("-netdev", netdev, "-device", virtio(builder.architecture, "net", "netdev=eth0")) return nil } @@ -587,7 +594,7 @@ func (builder *QemuBuilder) setupAdditionalNetworking() error { macSuffix := fmt.Sprintf("%02x", macCounter) netdev := fmt.Sprintf("user,id=eth%s,dhcpstart=10.0.2.%s", idSuffix, netSuffix) - device := virtio("net", fmt.Sprintf("netdev=eth%s,mac=52:55:00:d1:56:%s", idSuffix, macSuffix)) + device := virtio(builder.architecture, "net", fmt.Sprintf("netdev=eth%s,mac=52:55:00:d1:56:%s", idSuffix, macSuffix)) builder.Append("-netdev", netdev, "-device", device) macCounter++ } @@ -595,6 +602,16 @@ func (builder *QemuBuilder) setupAdditionalNetworking() error { return nil } +// SetArchitecture enables qemu full emulation for the target architecture. +func (builder *QemuBuilder) SetArchitecture(arch string) error { + switch arch { + case "x86_64", "aarch64", "s390x", "ppc64le": + builder.architecture = arch + return nil + } + return fmt.Errorf("architecture %s not supported by coreos-assembler qemu", arch) +} + // Mount9p sets up a mount point from the host to guest. To be replaced // with https://virtio-fs.gitlab.io/ once it lands everywhere. func (builder *QemuBuilder) Mount9p(source, destHint string, readonly bool) { @@ -604,13 +621,13 @@ func (builder *QemuBuilder) Mount9p(source, destHint string, readonly bool) { readonlyStr = ",readonly=on" } builder.Append("--fsdev", fmt.Sprintf("local,id=fs%d,path=%s,security_model=mapped%s", builder.fs9pID, source, readonlyStr)) - builder.Append("-device", virtio("9p", fmt.Sprintf("fsdev=fs%d,mount_tag=%s", builder.fs9pID, destHint))) + builder.Append("-device", virtio(builder.architecture, "9p", fmt.Sprintf("fsdev=fs%d,mount_tag=%s", builder.fs9pID, destHint))) } // supportsFwCfg if the target system supports injecting // Ignition via the qemu -fw_cfg option. func (builder *QemuBuilder) supportsFwCfg() bool { - switch coreosarch.CurrentRpmArch() { + switch builder.architecture { case "s390x", "ppc64le": return false } @@ -619,7 +636,7 @@ func (builder *QemuBuilder) supportsFwCfg() bool { // supportsSwtpm if the target system supports a virtual TPM device func (builder *QemuBuilder) supportsSwtpm() bool { - switch coreosarch.CurrentRpmArch() { + switch builder.architecture { case "s390x": // s390x does not support a backend for TPM return false @@ -655,7 +672,7 @@ type coreosGuestfish struct { remote string } -func newGuestfish(diskImagePath string, diskSectorSize int) (*coreosGuestfish, error) { +func newGuestfish(arch, diskImagePath string, diskSectorSize int) (*coreosGuestfish, error) { // Set guestfish backend to direct in order to avoid libvirt as backend. // Using libvirt can lead to permission denied issues if it does not have access // rights to the qcow image @@ -668,7 +685,7 @@ func newGuestfish(diskImagePath string, diskSectorSize int) (*coreosGuestfish, e cmd.Env = append(os.Environ(), "LIBGUESTFS_BACKEND=direct") // Hack to run with a wrapper on older P8 hardware running RHEL7 - switch coreosarch.CurrentRpmArch() { + switch arch { case "ppc64le": u := unix.Utsname{} if err := unix.Uname(&u); err != nil { @@ -736,8 +753,8 @@ func (gf *coreosGuestfish) destroy() { } // setupPreboot performs changes necessary before the disk is booted -func setupPreboot(confPath, firstbootkargs, kargs string, diskImagePath string, diskSectorSize int) error { - gf, err := newGuestfish(diskImagePath, diskSectorSize) +func setupPreboot(arch, confPath, firstbootkargs, kargs string, diskImagePath string, diskSectorSize int) error { + gf, err := newGuestfish(arch, diskImagePath, diskSectorSize) if err != nil { return err } @@ -898,7 +915,7 @@ func (builder *QemuBuilder) addDiskImpl(disk *Disk, primary bool) error { } requiresInjection := builder.ConfigFile != "" && builder.ForceConfigInjection if requiresInjection || builder.AppendFirstbootKernelArgs != "" || builder.AppendKernelArgs != "" { - if err := setupPreboot(builder.ConfigFile, builder.AppendFirstbootKernelArgs, builder.AppendKernelArgs, + if err := setupPreboot(builder.architecture, builder.ConfigFile, builder.AppendFirstbootKernelArgs, builder.AppendKernelArgs, disk.dstFileName, disk.SectorSize); err != nil { return errors.Wrapf(err, "ignition injection with guestfs failed") } @@ -952,13 +969,13 @@ func (builder *QemuBuilder) addDiskImpl(disk *Disk, primary bool) error { wwn := rand.Uint64() var bus string - switch coreosarch.CurrentRpmArch() { + switch builder.architecture { case "x86_64", "ppc64le", "aarch64": bus = "pci" case "s390x": bus = "ccw" default: - panic(fmt.Sprintf("Mantle doesn't know which bus type to use on %s", coreosarch.CurrentRpmArch())) + panic(fmt.Sprintf("Mantle doesn't know which bus type to use on %s", builder.architecture)) } for i := 0; i < 2; i++ { @@ -984,7 +1001,7 @@ func (builder *QemuBuilder) addDiskImpl(disk *Disk, primary bool) error { disk.dstFileName = "" switch channel { case "virtio": - builder.Append("-device", virtio("blk", fmt.Sprintf("drive=%s%s", id, opts))) + builder.Append("-device", virtio(builder.architecture, "blk", fmt.Sprintf("drive=%s%s", id, opts))) case "nvme": builder.Append("-device", fmt.Sprintf("nvme,drive=%s%s", id, opts)) default: @@ -1067,7 +1084,7 @@ func (builder *QemuBuilder) finalize() { // Then later, other non-x86_64 seemed to just copy that. memory := 1024 - switch coreosarch.CurrentRpmArch() { + switch builder.architecture { case "aarch64", "s390x", "ppc64le": memory = 2048 } @@ -1083,15 +1100,16 @@ func (builder *QemuBuilder) Append(args ...string) { // baseQemuArgs takes a board and returns the basic qemu // arguments needed for the current architecture. -func baseQemuArgs() []string { +func baseQemuArgs(arch string) ([]string, error) { accel := "accel=kvm" kvm := true - if _, ok := os.LookupEnv("COSA_NO_KVM"); ok { + hostArch := coreosarch.CurrentRpmArch() + if _, ok := os.LookupEnv("COSA_NO_KVM"); ok || hostArch != arch { accel = "accel=tcg" kvm = false } var ret []string - switch coreosarch.CurrentRpmArch() { + switch arch { case "x86_64": ret = []string{ "qemu-system-x86_64", @@ -1113,19 +1131,19 @@ func baseQemuArgs() []string { "-machine", "pseries,kvm-type=HV,vsmt=8,cap-fwnmi=off," + accel, } default: - panic(fmt.Sprintf("RpmArch %s combo not supported for qemu ", coreosarch.CurrentRpmArch())) + return nil, fmt.Errorf("architecture %s not supported for qemu", arch) } if kvm { ret = append(ret, "-cpu", "host") } else { - if coreosarch.CurrentRpmArch() == "x86_64" { + if arch == "x86_64" { // the default qemu64 CPU model does not support x86_64_v2 // causing crashes on EL9+ kernels // see https://bugzilla.redhat.com/show_bug.cgi?id=2060839 ret = append(ret, "-cpu", "Nehalem") } } - return ret + return ret, nil } func (builder *QemuBuilder) setupUefi(secureBoot bool) error { @@ -1289,7 +1307,7 @@ func (builder *QemuBuilder) setupIso() error { } builder.Append("-drive", "file="+builder.iso.path+",format=raw,if=none,readonly=on,id=installiso") if builder.isoAsDisk { - builder.Append("-device", virtio("blk", "drive=installiso"+bootindexStr)) + builder.Append("-device", virtio(builder.architecture, "blk", "drive=installiso"+bootindexStr)) } else { builder.Append("-device", "ide-cd,drive=installiso"+bootindexStr) } @@ -1378,7 +1396,10 @@ func (builder *QemuBuilder) Exec() (*QemuInstance, error) { } }() - argv := baseQemuArgs() + argv, err := baseQemuArgs(builder.architecture) + if err != nil { + return nil, err + } argv = append(argv, "-m", fmt.Sprintf("%d", builder.Memory)) if builder.Processors < 0 { @@ -1416,7 +1437,7 @@ func (builder *QemuBuilder) Exec() (*QemuInstance, error) { // We always provide a random source argv = append(argv, "-object", "rng-random,filename=/dev/urandom,id=rng0", - "-device", virtio("rng", "rng=rng0")) + "-device", virtio(builder.architecture, "rng", "rng=rng0")) if builder.UUID != "" { argv = append(argv, "-uuid", builder.UUID) } @@ -1518,7 +1539,7 @@ func (builder *QemuBuilder) Exec() (*QemuInstance, error) { } argv = append(argv, "-chardev", fmt.Sprintf("socket,id=chrtpm,path=%s", swtpmSock), "-tpmdev", "emulator,id=tpm0,chardev=chrtpm") // There are different device backends on each architecture - switch coreosarch.CurrentRpmArch() { + switch builder.architecture { case "x86_64": argv = append(argv, "-device", "tpm-tis,tpmdev=tpm0") case "aarch64": @@ -1560,6 +1581,7 @@ func (builder *QemuBuilder) Exec() (*QemuInstance, error) { argv = append(argv, builder.Argv...) inst.qemu = exec.Command(argv[0], argv[1:]...) + inst.architecture = builder.architecture cmd := inst.qemu.(*exec.ExecCmd) cmd.Stderr = os.Stderr diff --git a/src/deps.txt b/src/deps.txt index 6a355b2346..53e3e8111f 100644 --- a/src/deps.txt +++ b/src/deps.txt @@ -30,6 +30,8 @@ make git rpm-build # virt dependencies libguestfs-tools libguestfs-tools-c /usr/bin/qemu-img qemu-kvm swtpm +# And the main arch emulators for cross-arch testing +qemu-system-aarch64-core qemu-system-ppc-core qemu-system-s390x-core qemu-system-x86-core # Useful for moving files around rsync