make it work

Signed-off-by: Shahriyar Jalayeri <[email protected]>

pkg/dom0-ztools/efi/Makefile

@@ -29,4 +29,4 @@ clean:
 %.efi: %.so
 	objcopy -j .text -j .sdata -j .data -j .dynamic \
         -j .dynsym  -j .rel -j .rela -j .reloc \
-        --target=efi-app-$(ARCH) $^ $@
+        --target=efi-app-$(ARCH) $^ $@

pkg/dom0-ztools/efi/efi_run.sh

@@ -8,4 +8,4 @@ cp /hostfs/etc/ovmf/eve_efi.img /run/kvm/eve_efi.img
     -drive "file=/run/kvm/eve_efi.img,format=raw" \
     -smbios "type=11,value=${2}" \
     -net none \
-    -nographic
+    -nographic

pkg/dom0-ztools/efi/main.c

@@ -694,4 +694,4 @@ efi_main(
   }
 
   return EFI_SUCCESS;
-}
+}

pkg/dom0-ztools/efi/startup.nsh

@@ -1,3 +1,3 @@
 fs0:
 efi_eveutil.efi
-reset -s
+reset -s

pkg/pillar/cmd/domainmgr/domainmgr.go

@@ -1119,14 +1119,21 @@ func maybeRetryBoot(ctx *domainContext, status *types.DomainStatus) {
 	}
 	defer file.Close()
 
-	if err := hyper.Task(status).VirtualTPMSetup(status.DomainName, agentName, ctx.ps, warningTime, errorTime); err != nil {
+	extra := types.ExtraArgs{
+		AgentName: agentName,
+		Ps:        ctx.ps,
+		WarnTime:  warningTime,
+		ErrTime:   errorTime,
+	}
+
+	if err := hyper.Task(status).VirtualTPMSetup(status.DomainName, &extra); err != nil {
 		log.Errorf("Failed to setup virtual TPM for %s: %s", status.DomainName, err)
 		status.VirtualTPM = false
 	} else {
 		status.VirtualTPM = true
 	}
 
-	if err := hyper.Task(status).Setup(*status, *config, ctx.assignableAdapters, nil, file); err != nil {
+	if err := hyper.Task(status).Setup(*status, *config, ctx.assignableAdapters, nil, file, &extra); err != nil {
 		//it is retry, so omit error
 		log.Errorf("Failed to create DomainStatus from %+v: %s",
 			config, err)
@@ -1671,15 +1678,22 @@ func doActivate(ctx *domainContext, config types.DomainConfig,
 	}
 	defer file.Close()
 
-	if err := hyper.Task(status).VirtualTPMSetup(status.DomainName, agentName, ctx.ps, warningTime, errorTime); err != nil {
+	extra := types.ExtraArgs{
+		AgentName: agentName,
+		Ps:        ctx.ps,
+		WarnTime:  warningTime,
+		ErrTime:   errorTime,
+	}
+
+	if err := hyper.Task(status).VirtualTPMSetup(status.DomainName, &extra); err != nil {
 		log.Errorf("Failed to setup virtual TPM for %s: %s", status.DomainName, err)
 		status.VirtualTPM = false
 	} else {
 		status.VirtualTPM = true
 	}
 
 	globalConfig := agentlog.GetGlobalConfig(log, ctx.subGlobalConfig)
-	if err := hyper.Task(status).Setup(*status, config, ctx.assignableAdapters, globalConfig, file); err != nil {
+	if err := hyper.Task(status).Setup(*status, config, ctx.assignableAdapters, globalConfig, file, &extra); err != nil {
 		log.Errorf("Failed to create DomainStatus from %+v: %s",
 			config, err)
 		status.SetErrorNow(err.Error())

pkg/pillar/hypervisor/containerd.go

@@ -11,7 +11,6 @@ import (
 	"time"
 
 	"github.com/lf-edge/eve/pkg/pillar/containerd"
-	"github.com/lf-edge/eve/pkg/pillar/pubsub"
 	"github.com/lf-edge/eve/pkg/pillar/types"
 	"github.com/opencontainers/runtime-spec/specs-go"
 
@@ -101,7 +100,7 @@ func (ctx ctrdContext) setupSpec(status *types.DomainStatus, config *types.Domai
 }
 
 func (ctx ctrdContext) Setup(status types.DomainStatus, config types.DomainConfig,
-	aa *types.AssignableAdapters, globalConfig *types.ConfigItemValueMap, file *os.File) error {
+	aa *types.AssignableAdapters, globalConfig *types.ConfigItemValueMap, file *os.File, extra *types.ExtraArgs) error {
 	if status.OCIConfigDir == "" {
 		return logError("failed to run domain %s: not based on an OCI image", status.DomainName)
 	}
@@ -327,7 +326,7 @@ func (ctx ctrdContext) GetDomsCPUMem() (map[string]types.DomainMetric, error) {
 	return res, nil
 }
 
-func (ctx ctrdContext) VirtualTPMSetup(domainName, agentName string, ps *pubsub.PubSub, warnTime, errTime time.Duration) error {
+func (ctx ctrdContext) VirtualTPMSetup(domainName string, extra *types.ExtraArgs) error {
 	return fmt.Errorf("not implemented")
 }
 

pkg/pillar/hypervisor/kubevirt.go

@@ -22,7 +22,6 @@ import (
 	"time"
 
 	"github.com/lf-edge/eve/pkg/pillar/base"
-	"github.com/lf-edge/eve/pkg/pillar/pubsub"
 	"github.com/lf-edge/eve/pkg/pillar/types"
 
 	netattdefv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
@@ -185,7 +184,7 @@ func (ctx kubevirtContext) Task(status *types.DomainStatus) types.Task {
 
 // Use eve DomainConfig and DomainStatus and generate k3s VMI config or a Pod config
 func (ctx kubevirtContext) Setup(status types.DomainStatus, config types.DomainConfig,
-	aa *types.AssignableAdapters, globalConfig *types.ConfigItemValueMap, file *os.File) error {
+	aa *types.AssignableAdapters, globalConfig *types.ConfigItemValueMap, file *os.File, extra *types.ExtraArgs) error {
 
 	diskStatusList := status.DiskStatusList
 	domainName := status.DomainName
@@ -1362,7 +1361,7 @@ func (ctx kubevirtContext) PCISameController(id1 string, id2 string) bool {
 	return PCISameControllerGeneric(id1, id2)
 }
 
-func (ctx kubevirtContext) VirtualTPMSetup(domainName, agentName string, ps *pubsub.PubSub, warnTime, errTime time.Duration) error {
+func (ctx kubevirtContext) VirtualTPMSetup(domainName string, extra *types.ExtraArgs) error {
 	return fmt.Errorf("not implemented")
 }
 

pkg/pillar/hypervisor/kvm.go

@@ -4,9 +4,11 @@
 package hypervisor
 
 import (
+	"bytes"
 	"fmt"
 	"net"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"runtime"
 	"strings"
@@ -17,7 +19,6 @@ import (
 	zconfig "github.com/lf-edge/eve-api/go/config"
 	"github.com/lf-edge/eve/pkg/pillar/agentlog"
 	"github.com/lf-edge/eve/pkg/pillar/containerd"
-	"github.com/lf-edge/eve/pkg/pillar/pubsub"
 	"github.com/lf-edge/eve/pkg/pillar/types"
 	"github.com/lf-edge/eve/pkg/pillar/utils"
 	fileutils "github.com/lf-edge/eve/pkg/pillar/utils/file"
@@ -738,35 +739,18 @@ func getOVMFSettingsFilename(domainName string) (string, error) {
 	return types.OVMFSettingsDir + "/" + domainUUID.String() + "_OVMF_VARS.fd", nil
 }
 
-func prepareOVMFSettings(config types.DomainConfig, status types.DomainStatus, globalConfig *types.ConfigItemValueMap) error {
+func prepareOVMFSettings(status types.DomainStatus) error {
 	// Create the OVMF settings directory if it does not exist
 	if err := os.MkdirAll(types.OVMFSettingsDir, 0755); err != nil {
 		return logError("failed to create OVMF settings directory: %v", err)
 	}
-	// Create a copy of the ovmf_vars.bin file in <domainName>_ovmf_vars.bin
+	// Get the path for a copy of the ovmf_vars.bin file form of <domainName>_OVMF_VARS.fd
 	ovmfSettingsFile, err := getOVMFSettingsFilename(status.DomainName)
 	if err != nil {
 		return logError("failed to get OVMF settings file: %v", err)
 	}
-	// Check if we need custom OVMF settings for the domain (the resolution)
-	fmlResolution := types.FmlResolutionUnset
-	if config.VirtualizationMode == types.FML {
-		// if we are not getting the resolution from the cloud-init, check the
-		// global config.
-		fmlResolution = status.FmlCustomResolution
-		if fmlResolution == types.FmlResolutionUnset {
-			if fmlResolution, err = getFmlCustomResolution(&status, globalConfig); err != nil {
-				return logError("failed to get custom resolution for domain %s: %v", status.DomainName, err)
-			}
-		}
-	}
-	// Find the necessary OVMF settings file
-	ovmfSettingsFileSrc := types.OVMFSettingsTemplate
-	if fmlResolution != types.FmlResolutionUnset {
-		ovmfSettingsFileSrc = types.CustomOVMFSettingsDir + "/OVMF_VARS_" + fmlResolution + ".fd"
-	}
 	if _, err := os.Stat(ovmfSettingsFile); os.IsNotExist(err) {
-		if err := fileutils.CopyFile(ovmfSettingsFileSrc, ovmfSettingsFile); err != nil {
+		if err := fileutils.CopyFile(types.OVMFSettingsTemplate, ovmfSettingsFile); err != nil {
 			return logError("failed to copy OVMF_VARS file: %v", err)
 		}
 	}
@@ -790,7 +774,7 @@ func cleanupOVMFSettings(domainName string) error {
 
 // Setup sets up kvm
 func (ctx KvmContext) Setup(status types.DomainStatus, config types.DomainConfig,
-	aa *types.AssignableAdapters, globalConfig *types.ConfigItemValueMap, file *os.File) error {
+	aa *types.AssignableAdapters, globalConfig *types.ConfigItemValueMap, file *os.File, extra *types.ExtraArgs) error {
 
 	diskStatusList := status.DiskStatusList
 	domainName := status.DomainName
@@ -814,9 +798,29 @@ func (ctx KvmContext) Setup(status types.DomainStatus, config types.DomainConfig
 	// for ARM produces a single QEMU_EFI.fd file that contains both OVMF_VARS.fd
 	// and OVMF_CODE.fd.
 	if config.VirtualizationMode == types.FML && runtime.GOARCH == "amd64" {
-		if err := prepareOVMFSettings(config, status, globalConfig); err != nil {
+		// Copy the OVMF_VARS file and the eve_efi.img file to the domain directory
+		err := prepareOVMFSettings(status)
+		if err != nil {
 			return logError("failed to setup OVMF settings for domain %s: %v", status.DomainName, err)
 		}
+
+		fmlResolution := status.FmlCustomResolution
+		if fmlResolution == types.FmlResolutionUnset {
+			if fmlResolution, err = getFmlCustomResolution(&status, globalConfig); err != nil {
+				return logError("failed to get custom resolution for domain %s: %v", status.DomainName, err)
+			}
+		}
+		if fmlResolution != types.FmlResolutionUnset {
+			cfg := []string{
+				"eve.fml.resolution:" + fmlResolution,
+			}
+
+			logError("setOvmfVariables before call")
+			err := ctx.setOvmfVariables(cfg, status.DomainName, extra)
+			if err != nil {
+				return logError("failed to set OVMF variables for domain %s: %v", status.DomainName, err)
+			}
+		}
 	}
 
 	// first lets build the domain config
@@ -846,6 +850,14 @@ func (ctx KvmContext) Setup(status types.DomainStatus, config types.DomainConfig
 		"-readconfig", file.Name(),
 		"-pidfile", kvmStateDir+domainName+"/pid")
 
+	if config.VirtualizationMode == types.FML && runtime.GOARCH == "amd64" {
+		args = append(args, "-drive", "file=/run/kvm/eve_efi.img,format=raw,if=none,id=drive-eve-efi")
+		args = append(args, "-device", "virtio-blk-pci,drive=drive-eve-efi")
+		args = append(args, "-smbios", "type=11,value=eve.try.all.boot.options:true")
+	}
+
+	logError("guest args: %v", args)
+
 	spec, err := ctx.setupSpec(&status, &config, status.OCIConfigDir)
 
 	if err != nil {
@@ -1330,9 +1342,9 @@ func getQmpListenerSocket(domainName string) string {
 }
 
 // VirtualTPMSetup launches a vTPM instance for the domain
-func (ctx KvmContext) VirtualTPMSetup(domainName, agentName string, ps *pubsub.PubSub, warnTime, errTime time.Duration) error {
-	if ps != nil {
-		wk := utils.NewWatchdogKick(ps, agentName, warnTime, errTime)
+func (ctx KvmContext) VirtualTPMSetup(domainName string, extra *types.ExtraArgs) error {
+	if extra.Ps != nil {
+		wk := utils.NewWatchdogKick(extra.Ps, extra.AgentName, extra.WarnTime, extra.ErrTime)
 		domainUUID, _, _, err := types.DomainnameToUUID(domainName)
 		if err != nil {
 			return fmt.Errorf("failed to extract UUID from domain name (vTPM setup): %v", err)
@@ -1429,3 +1441,68 @@ func requestVtpmTermination(id uuid.UUID) error {
 
 	return nil
 }
+
+// This is janky and hacky, but it is OK for testing
+func (ctx KvmContext) setOvmfVariables(config []string, domainName string, extra *types.ExtraArgs) error {
+	cmd := exec.Command("/hostfs/usr/bin/ctr", "--namespace", "services.linuxkit", "t", "ls")
+	var out bytes.Buffer
+	cmd.Stdout = &out
+	if err := cmd.Run(); err != nil {
+		return logError("setOvmfVariables error running ctr: %s\n", err)
+	}
+	output := out.String()
+	lines := strings.Split(output, "\n")
+	var xenToolsPID string
+	for _, line := range lines {
+		columns := strings.Fields(line)
+		if len(columns) > 2 && columns[0] == "xen-tools" {
+			xenToolsPID = columns[1]
+			break
+		}
+	}
+	if xenToolsPID != "" {
+		logError("setOvmfVariables The PID of xen-tools is: %s\n", xenToolsPID)
+	} else {
+		return logError("setOvmfVariables xen-tools not found.")
+	}
+
+	ovmfSettingsFile, err := getOVMFSettingsFilename(domainName)
+	if err != nil {
+		return logError("setOvmfVariables failed to get OVMF settings file: %v", err)
+	}
+
+	args := []string{"-F", "-a", "-t", xenToolsPID,
+		"/hostfs/etc/ovmf/efi_run.sh",
+		ovmfSettingsFile,
+		"eve.fml.resolution:1920x1080",
+	}
+
+	logError("setOvmfVariables args: nsenter %v", args)
+	cmd = exec.Command("nsenter", args...)
+	if err := cmd.Start(); err != nil {
+		return logError("setOvmfVariables error starting nsenter: %s\n", err)
+	}
+	done := make(chan error, 1)
+	go func() {
+		done <- cmd.Wait()
+	}()
+
+waitLoop:
+	for {
+		select {
+		case err := <-done:
+			if err != nil {
+				logError("setOvmfVariables Process exited with error: %s\n", err)
+			} else {
+				logError("setOvmfVariables Process completed successfully.")
+			}
+			break waitLoop
+		default:
+			extra.Ps.StillRunning(extra.AgentName, extra.WarnTime, extra.ErrTime)
+			logError("setOvmfVariables still waiting.")
+			time.Sleep(1 * time.Second)
+		}
+	}
+
+	return nil
+}

pkg/pillar/hypervisor/null.go

@@ -6,9 +6,7 @@ package hypervisor
 import (
 	"fmt"
 	"os"
-	"time"
 
-	"github.com/lf-edge/eve/pkg/pillar/pubsub"
 	"github.com/lf-edge/eve/pkg/pillar/types"
 
 	uuid "github.com/satori/go.uuid"
@@ -69,7 +67,7 @@ func (ctx nullContext) Task(status *types.DomainStatus) types.Task {
 }
 
 func (ctx nullContext) Setup(types.DomainStatus, types.DomainConfig,
-	*types.AssignableAdapters, *types.ConfigItemValueMap, *os.File) error {
+	*types.AssignableAdapters, *types.ConfigItemValueMap, *os.File, *types.ExtraArgs) error {
 	return nil
 }
 
@@ -174,7 +172,7 @@ func (ctx nullContext) GetDomsCPUMem() (map[string]types.DomainMetric, error) {
 	return nil, nil
 }
 
-func (ctx nullContext) VirtualTPMSetup(domainName, agentName string, ps *pubsub.PubSub, warnTime, errTime time.Duration) error {
+func (ctx nullContext) VirtualTPMSetup(domainName string, extra *types.ExtraArgs) error {
 	return fmt.Errorf("not implemented")
 }
 

pkg/pillar/hypervisor/xen.go

@@ -11,9 +11,7 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
-	"time"
 
-	"github.com/lf-edge/eve/pkg/pillar/pubsub"
 	"github.com/lf-edge/eve/pkg/pillar/types"
 	"github.com/shirou/gopsutil/cpu"
 	"github.com/shirou/gopsutil/mem"
@@ -121,7 +119,7 @@ func (ctx xenContext) Task(status *types.DomainStatus) types.Task {
 }
 
 func (ctx xenContext) Setup(status types.DomainStatus, config types.DomainConfig,
-	aa *types.AssignableAdapters, globalConfig *types.ConfigItemValueMap, file *os.File) error {
+	aa *types.AssignableAdapters, globalConfig *types.ConfigItemValueMap, file *os.File, extra *types.ExtraArgs) error {
 	// first lets build the domain config
 	if err := ctx.CreateDomConfig(status.DomainName, config,
 		status.DiskStatusList, aa, globalConfig, file); err != nil {
@@ -877,7 +875,7 @@ func fallbackDomainMetric() map[string]types.DomainMetric {
 	return dmList
 }
 
-func (ctx xenContext) VirtualTPMSetup(domainName, agentName string, ps *pubsub.PubSub, warnTime, errTime time.Duration) error {
+func (ctx xenContext) VirtualTPMSetup(domainName string, extra *types.ExtraArgs) error {
 	return fmt.Errorf("not implemented")
 }