diff --git a/cmd/machine-config-daemon/mount_container.go b/cmd/machine-config-daemon/mount_container.go new file mode 100644 index 0000000000..0ef8ff9f47 --- /dev/null +++ b/cmd/machine-config-daemon/mount_container.go @@ -0,0 +1,71 @@ +package main + +import ( + "flag" + "fmt" + "os" + + daemon "github.com/openshift/machine-config-operator/pkg/daemon" + "github.com/openshift/machine-config-operator/pkg/daemon/constants" + "github.com/spf13/cobra" + "github.com/spf13/pflag" +) + +var mountContainer = &cobra.Command{ + Use: "mount-container", + DisableFlagsInUseLine: true, + Short: "Pull and mount container", + Args: cobra.ExactArgs(1), + Run: executeMountContainer, +} + +// init executes upon import +func init() { + rootCmd.AddCommand(mountContainer) + pflag.CommandLine.AddGoFlagSet(flag.CommandLine) +} + +func saveToFile(content, path string) error { + file, err := os.Create(path) + if err != nil { + file.Close() + return fmt.Errorf("Error creating file %s: %v", path, err) + } + defer file.Close() + if _, err := file.WriteString(content); err != nil { + return err + } + return nil + +} + +func runMountContainer(_ *cobra.Command, args []string) error { + flag.Set("logtostderr", "true") + flag.Parse() + var containerMntLoc, containerImage, containerName string + containerImage = args[0] + + var err error + if containerMntLoc, containerName, err = daemon.MountOSContainer(containerImage); err != nil { + return err + } + // Save mounted container name and location into file for later to be used + // for OS rebase and applying extensions + if err := saveToFile(containerName, constants.MountedOSContainerName); err != nil { + return fmt.Errorf("Failed saving container name: %v", err) + } + if err := saveToFile(containerMntLoc, constants.MountedOSContainerLocation); err != nil { + return fmt.Errorf("Failed saving mounted container location: %v", err) + } + + return nil + +} + +func executeMountContainer(cmd *cobra.Command, args []string) { + err := runMountContainer(cmd, args) + if err != nil { + fmt.Printf("error: %v\n", err) + os.Exit(1) + } +} diff --git a/cmd/machine-config-daemon/pivot.go b/cmd/machine-config-daemon/pivot.go index 48221cc9bd..cf5678c467 100644 --- a/cmd/machine-config-daemon/pivot.go +++ b/cmd/machine-config-daemon/pivot.go @@ -15,6 +15,7 @@ import ( "github.com/golang/glog" daemon "github.com/openshift/machine-config-operator/pkg/daemon" + "github.com/openshift/machine-config-operator/pkg/daemon/constants" "github.com/openshift/machine-config-operator/pkg/daemon/pivot/types" errors "github.com/pkg/errors" "github.com/spf13/cobra" @@ -214,8 +215,10 @@ func run(_ *cobra.Command, args []string) (retErr error) { flag.Set("logtostderr", "true") flag.Parse() - var container string + var containerName string if fromEtcPullSpec || len(args) == 0 { + // In this case we will just rebase to OSImage provided in etcPivotFile. + // Extensions won't apply. This should be consistent with old behavior. fromEtcPullSpec = true data, err := ioutil.ReadFile(etcPivotFile) if err != nil { @@ -224,14 +227,34 @@ func run(_ *cobra.Command, args []string) (retErr error) { } return errors.Wrapf(err, "failed to read from %s", etcPivotFile) } - container = strings.TrimSpace(string(data)) + container := strings.TrimSpace(string(data)) + unitName := "mco-mount-container" + glog.Infof("Pulling in image and mounting container on host via systemd-run unit=%s", unitName) + err = exec.Command("systemd-run", "--wait", "--collect", "--unit="+unitName, + "-E", "RPMOSTREE_CLIENT_ID=mco", constants.HostSelfBinary, "mount-container", container).Run() + if err != nil { + return errors.Wrapf(err, "failed to create extensions repo") + } + var containerName string + if containerName, err = daemon.ReadFromFile(constants.MountedOSContainerName); err != nil { + return err + } + + defer func() { + // Ideally other than MCD pivot, OSContainer shouldn't be needed by others. + // With above assumption, we should be able to delete OSContainer image as well as associated container with force option + exec.Command("podman", "rm", containerName).Run() + exec.Command("podman", "rmi", container).Run() + glog.Infof("Deleted container %s and corresponding image %s", containerName, container) + }() + } else { - container = args[0] + containerName = args[0] } client := daemon.NewNodeUpdaterClient() - _, changed, err := client.PullAndRebase(container, keep) + changed, err := client.PerformRpmOSTreeOperations(containerName, keep) if err != nil { return err } @@ -257,7 +280,7 @@ func run(_ *cobra.Command, args []string) (retErr error) { } if !changed { - glog.Info("No changes; already at target oscontainer, no kernel args provided") + glog.Info("No changes; already at target oscontainer, no kernel args provided, no kernelType switch, no extensions applied") } return nil diff --git a/install/0000_80_machine-config-operator_01_machineconfig.crd.yaml b/install/0000_80_machine-config-operator_01_machineconfig.crd.yaml index c35930c071..aa9be5c77b 100644 --- a/install/0000_80_machine-config-operator_01_machineconfig.crd.yaml +++ b/install/0000_80_machine-config-operator_01_machineconfig.crd.yaml @@ -307,6 +307,12 @@ spec: description: Name is the name of the unit. This must be suffixed with a valid unit type (e.g. 'thing.service') type: string + extensions: + description: List of additional features that can be enabled on host + type: array + items: + type: string + nullable: true fips: description: FIPS controls FIPS mode type: boolean diff --git a/lib/resourcemerge/machineconfig.go b/lib/resourcemerge/machineconfig.go index be81ef2a65..27a7f0b390 100644 --- a/lib/resourcemerge/machineconfig.go +++ b/lib/resourcemerge/machineconfig.go @@ -59,6 +59,10 @@ func ensureMachineConfigSpec(modified *bool, existing *mcfgv1.MachineConfigSpec, *modified = true (*existing).FIPS = required.FIPS } + if !equality.Semantic.DeepEqual(existing.Extensions, required.Extensions) { + *modified = true + (*existing).Extensions = required.Extensions + } } func ensureControllerConfigSpec(modified *bool, existing *mcfgv1.ControllerConfigSpec, required mcfgv1.ControllerConfigSpec) { diff --git a/pkg/apis/machineconfiguration.openshift.io/v1/types.go b/pkg/apis/machineconfiguration.openshift.io/v1/types.go index 30a6c60e3f..03633e71cb 100644 --- a/pkg/apis/machineconfiguration.openshift.io/v1/types.go +++ b/pkg/apis/machineconfiguration.openshift.io/v1/types.go @@ -176,6 +176,7 @@ type MachineConfigSpec struct { // +nullable KernelArguments []string `json:"kernelArguments"` + Extensions []string `json:"extensions"` FIPS bool `json:"fips"` KernelType string `json:"kernelType"` diff --git a/pkg/apis/machineconfiguration.openshift.io/v1/zz_generated.deepcopy.go b/pkg/apis/machineconfiguration.openshift.io/v1/zz_generated.deepcopy.go index 14e903d60d..45ed5a88de 100644 --- a/pkg/apis/machineconfiguration.openshift.io/v1/zz_generated.deepcopy.go +++ b/pkg/apis/machineconfiguration.openshift.io/v1/zz_generated.deepcopy.go @@ -678,6 +678,11 @@ func (in *MachineConfigSpec) DeepCopyInto(out *MachineConfigSpec) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.Extensions != nil { + in, out := &in.Extensions, &out.Extensions + *out = make([]string, len(*in)) + copy(*out, *in) + } return } diff --git a/pkg/controller/common/helpers.go b/pkg/controller/common/helpers.go index 316e8e47a4..667a4c7eed 100644 --- a/pkg/controller/common/helpers.go +++ b/pkg/controller/common/helpers.go @@ -103,6 +103,18 @@ func MergeMachineConfigs(configs []*mcfgv1.MachineConfig, osImageURL string) (*m kargs = append(kargs, cfg.Spec.KernelArguments...) } + extensions := []string{} + for _, cfg := range configs { + extensions = append(extensions, cfg.Spec.Extensions...) + } + + // Ensure that kernel-devel extension is applied only with default kernel. + if kernelType != KernelTypeDefault { + if InSlice("kernel-devel", extensions) { + return nil, fmt.Errorf("Installing kernel-devel extension is not supported with kernelType: %s", kernelType) + } + } + return &mcfgv1.MachineConfig{ Spec: mcfgv1.MachineConfigSpec{ OSImageURL: osImageURL, @@ -112,6 +124,7 @@ func MergeMachineConfigs(configs []*mcfgv1.MachineConfig, osImageURL string) (*m }, FIPS: fips, KernelType: kernelType, + Extensions: extensions, }, }, nil } @@ -285,12 +298,41 @@ func ValidateIgnition(ignconfig interface{}) error { } } +// InSlice search for an element in slice and return true if found, otherwise return false +func InSlice(elem string, slice []string) bool { + for _, k := range slice { + if k == elem { + return true + } + } + return false +} + +func validateExtensions(exts []string) error { + supportedExtensions := []string{"usbguard", "kernel-devel"} + invalidExts := []string{} + for _, ext := range exts { + if !InSlice(ext, supportedExtensions) { + invalidExts = append(invalidExts, ext) + } + } + if len(invalidExts) != 0 { + return fmt.Errorf("Invalid extensions found: %v", invalidExts) + } + return nil + +} + // ValidateMachineConfig validates that given MachineConfig Spec is valid. func ValidateMachineConfig(cfg mcfgv1.MachineConfigSpec) error { if !(cfg.KernelType == "" || cfg.KernelType == KernelTypeDefault || cfg.KernelType == KernelTypeRealtime) { return errors.Errorf("kernelType=%s is invalid", cfg.KernelType) } + if err := validateExtensions(cfg.Extensions); err != nil { + return err + } + if cfg.Config.Raw != nil { ignCfg, err := IgnParseWrapper(cfg.Config.Raw) if err != nil { diff --git a/pkg/daemon/constants/constants.go b/pkg/daemon/constants/constants.go index 746051be92..48a1bb7de3 100644 --- a/pkg/daemon/constants/constants.go +++ b/pkg/daemon/constants/constants.go @@ -55,4 +55,23 @@ const ( // "currentConfig" state. Create this file (empty contents is fine) if you wish the MCD // to proceed and attempt to "reconcile" to the new "desiredConfig" state regardless. MachineConfigDaemonForceFile = "/run/machine-config-daemon-force" + + // NewMachineConfigPath contains all the data from the desired MachineConfig + // except the spec/Config. We use this information and compare with old MachineConfig + // during pivot to update OS to desired state. + NewMachineConfigPath = "/run/newMachineConfig.json" + + // OldMachineConfigPath contains all the data from the MachineConfig from the current state + // except the spec/Config. We use this information and compare with new old MachienConfig + // during pivot to update OS to desired state. + OldMachineConfigPath = "/run/oldMachineConfig.json" + + // MountedOSContainerName contains name of OSContainer which we have already pulled in, created and mounted + // while we created extensions repo. We are saving the already created container so that we can further + // perform OS rebase or install extensions + MountedOSContainerName = "/run/mountedOSContainerName" + + // MountedOSContainerLocation contains the path of mounted container referenced in + // MountedOSContainer + MountedOSContainerLocation = "/run/mountedOSContainerLocation" ) diff --git a/pkg/daemon/rpm-ostree.go b/pkg/daemon/rpm-ostree.go index ad50c3d8b6..29cd4d6fb4 100644 --- a/pkg/daemon/rpm-ostree.go +++ b/pkg/daemon/rpm-ostree.go @@ -5,16 +5,14 @@ import ( "fmt" "os" "os/exec" + "reflect" "strings" "time" "github.com/golang/glog" - "github.com/opencontainers/go-digest" + mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" "github.com/openshift/machine-config-operator/pkg/daemon/constants" - pivottypes "github.com/openshift/machine-config-operator/pkg/daemon/pivot/types" - pivotutils "github.com/openshift/machine-config-operator/pkg/daemon/pivot/utils" "github.com/pkg/errors" - "k8s.io/apimachinery/pkg/util/uuid" ) const ( @@ -45,19 +43,25 @@ type RpmOstreeDeployment struct { RequestedLocalPkgs []string `json:"requested-local-packages"` } -// imageInspection is a public implementation of -// https://github.com/containers/skopeo/blob/82186b916faa9c8c70cfa922229bafe5ae024dec/cmd/skopeo/inspect.go#L20-L31 -type imageInspection struct { - Name string `json:",omitempty"` - Tag string `json:",omitempty"` - Digest digest.Digest - RepoDigests []string - Created *time.Time - DockerVersion string - Labels map[string]string - Architecture string - Os string - Layers []string +// imageInspection is motivated from podman upstream podman inspect +// https://github.com/containers/podman/blob/master/pkg/inspect/inspect.go#L13 +type containerInspection struct { + Name string + ImageName string + Created *time.Time + Config *ImageConfig + GraphDriver *Data +} + +// Data handles the data for a storage driver +type Data struct { + Name string + Data map[string]string +} + +// ImageConfig defines the execution parameters which should be used as a base when running a container using an image. +type ImageConfig struct { + Labels map[string]string } // NodeUpdaterClient is an interface describing how to interact with the host @@ -65,9 +69,10 @@ type imageInspection struct { type NodeUpdaterClient interface { GetStatus() (string, error) GetBootedOSImageURL() (string, string, error) - PullAndRebase(string, bool) (string, bool, error) RunPivot(string) error + Rebase(string) (bool, error) GetBootedDeployment() (*RpmOstreeDeployment, error) + PerformRpmOSTreeOperations(string, bool) (bool, error) } // RpmOstreeClient provides all RpmOstree related methods in one structure. @@ -133,122 +138,191 @@ func (r *RpmOstreeClient) GetBootedOSImageURL() (string, string, error) { return osImageURL, bootedDeployment.Version, nil } -// podmanRemove kills and removes a container -func podmanRemove(cid string) { - // Ignore errors here - exec.Command("podman", "kill", cid).Run() - exec.Command("podman", "rm", "-f", cid).Run() +func readMachineConfigs() (*mcfgv1.MachineConfig, *mcfgv1.MachineConfig, error) { + var oldCfgFile, newCfgFile *os.File + var oldCfg, newCfg *mcfgv1.MachineConfig + var err error + + if oldCfgFile, err = os.Open(constants.OldMachineConfigPath); err != nil { + return nil, nil, err + } + defer oldCfgFile.Close() + decoder := json.NewDecoder(oldCfgFile) + if err := decoder.Decode(&oldCfg); err != nil { + return nil, nil, err + } + + if newCfgFile, err = os.Open(constants.NewMachineConfigPath); err != nil { + return nil, nil, err + } + defer newCfgFile.Close() + decoder = json.NewDecoder(newCfgFile) + if err := decoder.Decode(&newCfg); err != nil { + return nil, nil, err + } + + return oldCfg, newCfg, nil } -// PullAndRebase potentially rebases system if not already rebased. -func (r *RpmOstreeClient) PullAndRebase(container string, keep bool) (imgid string, changed bool, err error) { - defaultDeployment, err := r.GetBootedDeployment() - if err != nil { - return +func generateExtensionsArgs(oldConfig, newConfig *mcfgv1.MachineConfig) []string { + removed := []string{} + added := []string{} + + oldExt := make(map[string]bool) + for _, ext := range oldConfig.Spec.Extensions { + oldExt[ext] = true + } + newExt := make(map[string]bool) + for _, ext := range newConfig.Spec.Extensions { + newExt[ext] = true } - previousPivot := "" - if len(defaultDeployment.CustomOrigin) > 0 { - if strings.HasPrefix(defaultDeployment.CustomOrigin[0], "pivot://") { - previousPivot = defaultDeployment.CustomOrigin[0][len("pivot://"):] - glog.Infof("Previous pivot: %s", previousPivot) - } else { - glog.Infof("Previous custom origin: %s", defaultDeployment.CustomOrigin[0]) + for ext := range oldExt { + if !newExt[ext] { + removed = append(removed, ext) + } + } + for ext := range newExt { + if !oldExt[ext] { + added = append(added, ext) } - } else { - glog.Info("Current origin is not custom") } - var authArgs []string - if _, err := os.Stat(kubeletAuthFile); err == nil { - authArgs = append(authArgs, "--authfile", kubeletAuthFile) + extArgs := []string{"update"} + for _, ext := range added { + extArgs = append(extArgs, "--install", ext) + } + for _, ext := range removed { + extArgs = append(extArgs, "--uninstall", ext) } - // If we're passed a non-canonical image, resolve it to its sha256 now - isCanonicalForm := true - if _, err = getRefDigest(container); err != nil { - isCanonicalForm = false - // In non-canonical form, we pull unconditionally right now - args := []string{"pull", "-q"} - args = append(args, authArgs...) - args = append(args, container) - _, err = pivotutils.RunExt(numRetriesNetCommands, "podman", args...) - if err != nil { - return - } - } else { - if previousPivot != "" { - var targetMatched bool - targetMatched, err = compareOSImageURL(previousPivot, container) - if err != nil { - return - } - if targetMatched { - changed = false + return extArgs +} + +// applyExtensions processes specified extensions if it is supported on RHCOS. +// It deletes an extension if it is no logner available in new rendered MachineConfig. +// Newly requested extension will be installed and existing extensions gets updated +// if we have a new version of the extension available in machine-os-content. +func applyExtensions(oldConfig, newConfig *mcfgv1.MachineConfig) error { + extensionsEmpty := len(oldConfig.Spec.Extensions) == 0 && len(newConfig.Spec.Extensions) == 0 + if (extensionsEmpty) || + (reflect.DeepEqual(oldConfig.Spec.Extensions, newConfig.Spec.Extensions) && oldConfig.Spec.OSImageURL == newConfig.Spec.OSImageURL) { + return nil + } + args := generateExtensionsArgs(oldConfig, newConfig) + glog.Infof("Applying extensions : %+q", args) + if err := exec.Command("rpm-ostree", args...).Run(); err != nil { + return fmt.Errorf("Failed to execute rpm-ostree %+q : %v", args, err) + } + + return nil +} + +// PerformRpmOSTreeOperations runs all rpm-ostree related operations +func (r *RpmOstreeClient) PerformRpmOSTreeOperations(containerName string, keep bool) (changed bool, err error) { + var oldConfig, newConfig *mcfgv1.MachineConfig + // Check if we have reached here through update() or by directly calling m-c-d pivot + // In the latter case MachineConfig won't get written on host. In such situation let's just Rebase() + // and return back. Checking just oldconfig should be enough. + if _, err = os.Stat(constants.OldMachineConfigPath); err != nil { + if os.IsNotExist(err) { + glog.Infof("m-c-d pivot got called outside of MCO: Updating OS") + if changed, err = r.Rebase(containerName); err != nil { return } } + return + } - // Pull the image - args := []string{"pull", "-q"} - args = append(args, authArgs...) - args = append(args, container) - _, err = pivotutils.RunExt(numRetriesNetCommands, "podman", args...) - if err != nil { + if oldConfig, newConfig, err = readMachineConfigs(); err != nil { + return + } + + // rebase needs to be done before extension since rpm-ostree fails to rebase on top of applied extensions + // during firstboot test. Maybe related to https://discussion.fedoraproject.org/t/bus-owner-changed-aborting-when-trying-to-upgrade/1919/ + if oldConfig.Spec.OSImageURL != newConfig.Spec.OSImageURL { + glog.Infof("Updating OS") + if changed, err = r.Rebase(containerName); err != nil { + return + } + } + + // We can remove this check on FCOS when it starts shipping extensions repo + var operatingSystem string + if operatingSystem, err = GetHostRunningOS(); err != nil { + err = errors.Wrapf(err, "failed to fetch Running OS on host") + return + } + if operatingSystem == MachineConfigDaemonOSRHCOS { + // Apply and update extensions + if err = applyExtensions(oldConfig, newConfig); err != nil { + return + } + + // Switch to real time kernel + if err = switchKernel(oldConfig, newConfig); err != nil { return } } - inspectArgs := []string{"inspect", "--type=image"} - inspectArgs = append(inspectArgs, fmt.Sprintf("%s", container)) + return + +} + +func inspectContainer(containerName string) (*containerInspection, error) { + inspectArgs := []string{"inspect", "--type=container"} + inspectArgs = append(inspectArgs, fmt.Sprintf("%s", containerName)) var output []byte - output, err = runGetOut("podman", inspectArgs...) + output, err := runGetOut("podman", inspectArgs...) if err != nil { - return + return nil, err } - var imagedataArray []imageInspection + var imagedataArray []containerInspection err = json.Unmarshal(output, &imagedataArray) if err != nil { err = errors.Wrapf(err, "unmarshaling podman inspect") - return - } - imagedata := imagedataArray[0] - if !isCanonicalForm { - imgid = imagedata.RepoDigests[0] - glog.Infof("Resolved to: %s", imgid) - } else { - imgid = container + return nil, err } + return &imagedataArray[0], nil - containerName := pivottypes.PivotNamePrefix + string(uuid.NewUUID()) +} - // `podman mount` wants a container, so let's make create a dummy one, but not run it - var cidBuf []byte - cidBuf, err = runGetOut("podman", "create", "--net=none", "--annotation=org.openshift.machineconfigoperator.pivot=true", "--name", containerName, imgid) +// Rebase potentially rebases system if not already rebased. +func (r *RpmOstreeClient) Rebase(containerName string) (changed bool, err error) { + defaultDeployment, err := r.GetBootedDeployment() if err != nil { return } - defer func() { - // Kill our dummy container - podmanRemove(containerName) - }() - cid := strings.TrimSpace(string(cidBuf)) - // Use the container ID to find its mount point - var mntBuf []byte - mntBuf, err = runGetOut("podman", "mount", cid) - if err != nil { + previousPivot := "" + if len(defaultDeployment.CustomOrigin) > 0 { + if strings.HasPrefix(defaultDeployment.CustomOrigin[0], "pivot://") { + previousPivot = defaultDeployment.CustomOrigin[0][len("pivot://"):] + glog.Infof("Previous pivot: %s", previousPivot) + } else { + glog.Infof("Previous custom origin: %s", defaultDeployment.CustomOrigin[0]) + } + } else { + glog.Info("Current origin is not custom") + } + + var imagedata *containerInspection + if imagedata, err = inspectContainer(containerName); err != nil { return } - mnt := strings.TrimSpace(string(mntBuf)) - repo := fmt.Sprintf("%s/srv/repo", mnt) + + containerImageName := imagedata.ImageName + glog.Infof("Container Image is : %s", containerImageName) + + repo := fmt.Sprintf("%s/srv/repo", imagedata.GraphDriver.Data["MergedDir"]) + glog.Infof("Mounted Dir %s", imagedata.GraphDriver.Data["MergedDir"]) // Now we need to figure out the commit to rebase to // Commit label takes priority - ostreeCsum, ok := imagedata.Labels["com.coreos.ostree-commit"] + ostreeCsum, ok := imagedata.Config.Labels["com.coreos.ostree-commit"] if ok { - if ostreeVersion, ok := imagedata.Labels["version"]; ok { + if ostreeVersion, ok := imagedata.Config.Labels["version"]; ok { glog.Infof("Pivoting to: %s (%s)", ostreeVersion, ostreeCsum) } else { glog.Infof("Pivoting to: %s", ostreeCsum) @@ -279,8 +353,10 @@ func (r *RpmOstreeClient) PullAndRebase(container string, keep bool) (imgid stri } } + glog.Infof("Updating OS to %s", containerImageName) + // This will be what will be displayed in `rpm-ostree status` as the "origin spec" - customURL := fmt.Sprintf("pivot://%s", imgid) + customURL := fmt.Sprintf("pivot://%s", containerImageName) // RPM-OSTree can now directly slurp from the mounted container! // https://github.com/projectatomic/rpm-ostree/pull/1732 @@ -292,12 +368,6 @@ func (r *RpmOstreeClient) PullAndRebase(container string, keep bool) (imgid stri return } - // By default, delete the image. - if !keep { - // Related: https://github.com/containers/libpod/issues/2234 - exec.Command("podman", "rmi", imgid).Run() - } - changed = true return } @@ -308,7 +378,7 @@ func (r *RpmOstreeClient) PullAndRebase(container string, keep bool) (imgid stri // see https://github.com/openshift/pivot/pull/31 and // https://github.com/openshift/machine-config-operator/issues/314 // Basically rpm_ostree_t has mac_admin, container_t doesn't. -func (r *RpmOstreeClient) RunPivot(osImageURL string) error { +func (r *RpmOstreeClient) RunPivot(containerName string) error { journalStopCh := make(chan time.Time) defer close(journalStopCh) go followPivotJournalLogs(journalStopCh) @@ -329,7 +399,7 @@ func (r *RpmOstreeClient) RunPivot(osImageURL string) error { unitName := "mco-pivot" glog.Infof("Executing OS update (pivot) on host via systemd-run unit=%s", unitName) err := exec.Command("systemd-run", "--wait", "--collect", "--unit="+unitName, - "-E", "RPMOSTREE_CLIENT_ID=mco", constants.HostSelfBinary, "pivot", osImageURL).Run() + "-E", "RPMOSTREE_CLIENT_ID=mco", constants.HostSelfBinary, "pivot", containerName).Run() if err != nil { return errors.Wrapf(err, "failed to run pivot") } diff --git a/pkg/daemon/rpm-ostree_test.go b/pkg/daemon/rpm-ostree_test.go index e4f433044d..8496c11869 100644 --- a/pkg/daemon/rpm-ostree_test.go +++ b/pkg/daemon/rpm-ostree_test.go @@ -41,9 +41,14 @@ func (r RpmOstreeClientMock) RunPivot(string) error { return err } -// PullAndRebase is a mock -func (r RpmOstreeClientMock) PullAndRebase(string, bool) (string, bool, error) { - return "", false, nil +// Rebase is a mock +func (r RpmOstreeClientMock) Rebase(string) (bool, error) { + return false, nil +} + +// PerformRpmOSTreeOperations is a mock +func (r RpmOstreeClientMock) PerformRpmOSTreeOperations(string, bool) (bool, error) { + return false, nil } func (r RpmOstreeClientMock) GetStatus() (string, error) { diff --git a/pkg/daemon/update.go b/pkg/daemon/update.go index c052e099ea..5d4f5beb37 100644 --- a/pkg/daemon/update.go +++ b/pkg/daemon/update.go @@ -10,7 +10,6 @@ import ( "os/user" "path/filepath" "reflect" - "regexp" "strconv" "strings" "syscall" @@ -46,7 +45,8 @@ const ( // SSH Keys for user "core" will only be written at /home/core/.ssh coreUserSSHPath = "/home/core/.ssh/" // fipsFile is the file to check if FIPS is enabled - fipsFile = "/proc/sys/crypto/fips_enabled" + fipsFile = "/proc/sys/crypto/fips_enabled" + extensionsRepo = "/etc/yum.repos.d/coreos-extensions.repo" ) func installedRTKernelRpmsOnHost() ([]string, error) { @@ -216,29 +216,6 @@ func canonicalizeEmptyMC(config *mcfgv1.MachineConfig) *mcfgv1.MachineConfig { } } -// Returns true if updated packages or new RT kernel related packages are available -func rtKernelUpdateAvailable(updateRpms []os.FileInfo, installedRTKernelRpms []string) bool { - // if list of RT kernel packages in update is more than installed list, then we have additional packages to install - if len(updateRpms) > len(installedRTKernelRpms) { - return true - } - for _, pkg := range installedRTKernelRpms { - found := false - searchRpm := pkg + ".rpm" - for _, rpm := range updateRpms { - if rpm.Name() == searchRpm { - found = true - break - } - } - if !found { - return true - } - } - - return false -} - // return true if the machineConfigDiff is not empty func (dn *Daemon) compareMachineConfig(oldConfig, newConfig *mcfgv1.MachineConfig) (bool, error) { oldConfig = canonicalizeEmptyMC(oldConfig) @@ -255,7 +232,33 @@ func (dn *Daemon) compareMachineConfig(oldConfig, newConfig *mcfgv1.MachineConfi return true, nil } +func writeMachineConfigs(oldConfig, newConfig *mcfgv1.MachineConfig) error { + oldMCFile, err := os.Create(constants.OldMachineConfigPath) + if err != nil { + return fmt.Errorf("Can't open file: %v", err) + } + defer oldMCFile.Close() + + encoder := json.NewEncoder(oldMCFile) + if err := encoder.Encode(oldConfig); err != nil { + return fmt.Errorf("Error while encoding: %v", err) + } + + newMCFile, err := os.Create(constants.NewMachineConfigPath) + if err != nil { + return fmt.Errorf("Can't open file: %v", err) + } + defer newMCFile.Close() + + encoder = json.NewEncoder(newMCFile) + if err := encoder.Encode(newConfig); err != nil { + return fmt.Errorf("Error while encoding: %v", err) + } + return nil +} + // update the node to the provided node configuration. +//nolint:gocyclo func (dn *Daemon) update(oldConfig, newConfig *mcfgv1.MachineConfig) (retErr error) { oldConfig = canonicalizeEmptyMC(oldConfig) @@ -299,6 +302,20 @@ func (dn *Daemon) update(oldConfig, newConfig *mcfgv1.MachineConfig) (retErr err return errors.Wrapf(errUnreconcilable, "%v", wrappedErr) } + // Serialize old and new MachineConfigs in file to be processed later by rpm-ostree during pivot + if err := writeMachineConfigs(oldConfig, newConfig); err != nil { + return fmt.Errorf("Error serialzing MachineConfigs on host: %v", err) + } + + defer func() { + if retErr != nil { + if err := writeMachineConfigs(newConfig, oldConfig); err != nil { + retErr = errors.Wrapf(retErr, "error rolling back saved MachineConfig on disk %v", err) + return + } + } + }() + dn.logSystem("Starting update from %s to %s: %+v", oldConfigName, newConfigName, diff) if err := dn.drain(); err != nil { @@ -366,21 +383,15 @@ func (dn *Daemon) update(oldConfig, newConfig *mcfgv1.MachineConfig) (retErr err } }() - // Switch to real time kernel - if err := dn.switchKernel(oldConfig, newConfig); err != nil { + // OS update is needed only when OSImageURL or KernelType or Extensions has changed + mcDiff, err := newMachineConfigDiff(oldConfig, newConfig) + if err != nil { return err } - - defer func() { - if retErr != nil { - if err := dn.switchKernel(newConfig, oldConfig); err != nil { - retErr = errors.Wrapf(retErr, "error rolling back Real time Kernel %v", err) - return - } - } - }() - - return dn.updateOSAndReboot(newConfig) + if mcDiff.osUpdate || mcDiff.extensions || mcDiff.kernelType { + return dn.updateOSAndReboot(newConfig) + } + return dn.finalizeAndReboot(newConfig) } // machineConfigDiff represents an ad-hoc difference between two MachineConfig objects. @@ -395,6 +406,7 @@ type machineConfigDiff struct { files bool units bool kernelType bool + extensions bool } // isEmpty returns true if the machineConfigDiff has no changes, or @@ -429,6 +441,7 @@ func newMachineConfigDiff(oldConfig, newConfig *mcfgv1.MachineConfig) (*machineC // Both nil and empty slices are of zero length, // consider them as equal while comparing KernelArguments in both MachineConfigs kargsEmpty := len(oldConfig.Spec.KernelArguments) == 0 && len(newConfig.Spec.KernelArguments) == 0 + extensionsEmpty := len(oldConfig.Spec.Extensions) == 0 && len(newConfig.Spec.Extensions) == 0 return &machineConfigDiff{ osUpdate: oldConfig.Spec.OSImageURL != newConfig.Spec.OSImageURL, @@ -438,6 +451,7 @@ func newMachineConfigDiff(oldConfig, newConfig *mcfgv1.MachineConfig) (*machineC files: !reflect.DeepEqual(oldIgn.Storage.Files, newIgn.Storage.Files), units: !reflect.DeepEqual(oldIgn.Systemd.Units, newIgn.Systemd.Units), kernelType: canonicalizeKernelType(oldConfig.Spec.KernelType) != canonicalizeKernelType(newConfig.Spec.KernelType), + extensions: !(extensionsEmpty || reflect.DeepEqual(oldConfig.Spec.Extensions, newConfig.Spec.Extensions)), }, nil } @@ -652,15 +666,6 @@ func parseKernelArguments(kargs []string) []string { return parsed } -func inArray(elem string, array []string) bool { - for _, k := range array { - if k == elem { - return true - } - } - return false -} - // generateKargsCommand performs a diff between the old/new MC kernelArguments, // and generates the command line arguments suitable for `rpm-ostree kargs`. // Note what we really should be doing though is also looking at the *current* @@ -671,12 +676,12 @@ func generateKargsCommand(oldConfig, newConfig *mcfgv1.MachineConfig) []string { newKargs := parseKernelArguments(newConfig.Spec.KernelArguments) cmdArgs := []string{} for _, arg := range oldKargs { - if !inArray(arg, newKargs) { + if !ctrlcommon.InSlice(arg, newKargs) { cmdArgs = append(cmdArgs, "--delete="+arg) } } for _, arg := range newKargs { - if !inArray(arg, oldKargs) { + if !ctrlcommon.InSlice(arg, oldKargs) { cmdArgs = append(cmdArgs, "--append="+arg) } } @@ -698,8 +703,8 @@ func (dn *Daemon) updateKernelArguments(oldConfig, newConfig *mcfgv1.MachineConf return exec.Command("rpm-ostree", args...).Run() } -// mountOSContainer mounts the container and returns the mountpoint -func (dn *Daemon) mountOSContainer(container string) (mnt, containerName string, err error) { +// MountOSContainer mounts the container and returns the mountpoint +func MountOSContainer(container string) (mnt, containerName string, err error) { var authArgs []string if _, err = os.Stat(kubeletAuthFile); err == nil { authArgs = append(authArgs, "--authfile", kubeletAuthFile) @@ -728,25 +733,23 @@ func (dn *Daemon) mountOSContainer(container string) (mnt, containerName string, return } mnt = strings.TrimSpace(string(mntBuf)) + glog.Infof("Container ID, mnt and name %s %s %s\n", cid, mnt, containerName) return } // switchKernel updates kernel on host with the kernelType specified in MachineConfig. // Right now it supports default (traditional) and realtime kernel -func (dn *Daemon) switchKernel(oldConfig, newConfig *mcfgv1.MachineConfig) error { +func switchKernel(oldConfig, newConfig *mcfgv1.MachineConfig) error { // Do nothing if both old and new KernelType are of type default if canonicalizeKernelType(oldConfig.Spec.KernelType) == ctrlcommon.KernelTypeDefault && canonicalizeKernelType(newConfig.Spec.KernelType) == ctrlcommon.KernelTypeDefault { return nil } - // We support Kernel update only on RHCOS nodes - if dn.OperatingSystem != MachineConfigDaemonOSRHCOS { - return fmt.Errorf("Updating kernel on non-RHCOS nodes is not supported") - } defaultKernel := []string{"kernel", "kernel-core", "kernel-modules", "kernel-modules-extra"} + realtimeKernel := []string{"kernel-rt-core", "kernel-rt-modules", "kernel-rt-modules-extra", "kernel-rt-kvm"} var args []string - dn.logSystem("Initiating switch from kernel %s to %s", canonicalizeKernelType(oldConfig.Spec.KernelType), canonicalizeKernelType(newConfig.Spec.KernelType)) + glog.Infof("Initiating switch from kernel %s to %s", canonicalizeKernelType(oldConfig.Spec.KernelType), canonicalizeKernelType(newConfig.Spec.KernelType)) if canonicalizeKernelType(oldConfig.Spec.KernelType) == ctrlcommon.KernelTypeRealtime && canonicalizeKernelType(newConfig.Spec.KernelType) == ctrlcommon.KernelTypeDefault { var installedRTKernelRpms []string @@ -762,82 +765,33 @@ func (dn *Daemon) switchKernel(oldConfig, newConfig *mcfgv1.MachineConfig) error for _, installedRTKernelRpm := range installedRTKernelRpms { args = append(args, "--uninstall", installedRTKernelRpm) } - dn.logSystem("Switching to kernelType=%s, invoking rpm-ostree %+q", newConfig.Spec.KernelType, args) + glog.Infof("Switching to kernelType=%s, invoking rpm-ostree %+q", newConfig.Spec.KernelType, args) if err := exec.Command("rpm-ostree", args...).Run(); err != nil { return fmt.Errorf("Failed to execute rpm-ostree %+q : %v", args, err) } return nil } - var mnt, containerName string - var err error - if mnt, containerName, err = dn.mountOSContainer(newConfig.Spec.OSImageURL); err != nil { - return err - } - - defer func() { - // Delete container and remove image once we are done with using rpms available in OSContainer - podmanRemove(containerName) - exec.Command("podman", "rmi", newConfig.Spec.OSImageURL).Run() - dn.logSystem("Deleted container and removed OSContainer image") - }() - - // Get kernel-rt packages from OSContainer - rtRegex := regexp.MustCompile("kernel-rt(.*).rpm") - files, err := ioutil.ReadDir(mnt) - if err != nil { - return err - } - - rtKernelRpms := []os.FileInfo{} - for _, file := range files { - if rtRegex.MatchString(file.Name()) { - rtKernelRpms = append(rtKernelRpms, file) - } - } - - if len(rtKernelRpms) == 0 { - // No kernel-rt rpm package found - return fmt.Errorf("No kernel-rt package available in the OSContainer with URL %s", newConfig.Spec.OSImageURL) - } - if canonicalizeKernelType(oldConfig.Spec.KernelType) == ctrlcommon.KernelTypeDefault && canonicalizeKernelType(newConfig.Spec.KernelType) == ctrlcommon.KernelTypeRealtime { // Switch to RT kernel args = []string{"override", "remove"} args = append(args, defaultKernel...) - for _, rpm := range rtKernelRpms { - args = append(args, "--install", fmt.Sprintf("%s/%s", mnt, rpm.Name())) + for _, pkg := range realtimeKernel { + args = append(args, "--install", pkg) } - dn.logSystem("Switching to kernelType=%s, invoking rpm-ostree %+q", newConfig.Spec.KernelType, args) + glog.Infof("Switching to kernelType=%s, invoking rpm-ostree %+q", newConfig.Spec.KernelType, args) if err := exec.Command("rpm-ostree", args...).Run(); err != nil { return fmt.Errorf("Failed to execute rpm-ostree %+q : %v", args, err) } + return nil } if canonicalizeKernelType(oldConfig.Spec.KernelType) == ctrlcommon.KernelTypeRealtime && canonicalizeKernelType(newConfig.Spec.KernelType) == ctrlcommon.KernelTypeRealtime { if oldConfig.Spec.OSImageURL != newConfig.Spec.OSImageURL { - var installedRTKernelRpms []string - var err error - args = []string{"uninstall"} - if installedRTKernelRpms, err = installedRTKernelRpmsOnHost(); err != nil { - return fmt.Errorf("Error while fetching installed RT kernel on host %v", err) - } - if len(installedRTKernelRpms) == 0 { - return fmt.Errorf("No kernel-rt package installed on host") - } - for _, installedRTKernelRpm := range installedRTKernelRpms { - args = append(args, installedRTKernelRpm) - } - // Perform kernel-rt package update only if updated packages are available - if rtKernelUpdateAvailable(rtKernelRpms, installedRTKernelRpms) { - for _, rpm := range rtKernelRpms { - args = append(args, "--install", fmt.Sprintf("%s/%s", mnt, rpm.Name())) - } - dn.logSystem("Updating rt-kernel packages on host: %+q", args) - if err := exec.Command("rpm-ostree", args...).Run(); err != nil { - return fmt.Errorf("Failed to execute rpm-ostree %+q : %v", args, err) - } + glog.Infof("Updating rt-kernel packages on host: %+q", args) + if err := exec.Command("rpm-ostree", "update").Run(); err != nil { + return fmt.Errorf("Failed to execute rpm-ostree %+q : %v", args, err) } } } @@ -1298,6 +1252,33 @@ func (dn *Daemon) updateSSHKeys(newUsers []ign3types.PasswdUser) error { return nil } +func addExtRepo(mnt string) error { + if err := os.MkdirAll(filepath.Dir(extensionsRepo), 0755); err != nil { + return fmt.Errorf("error creating yum repo directory %s: %v", filepath.Dir(extensionsRepo), err) + } + repo, err := os.Create(extensionsRepo) + if err != nil { + return fmt.Errorf("error creating extensions repo %s: %v", extensionsRepo, err) + } + defer repo.Close() + if _, err := repo.WriteString("[coreos-extensions]\nenabled=1\nmetadata_expire=1m\nbaseurl=" + mnt + "/extensions/\ngpgcheck=0\nskip_if_unavailable=False\n"); err != nil { + return err + } + return nil +} + +// ReadFromFile reads the content from a file and returns the content as string +func ReadFromFile(file string) (string, error) { + if _, err := os.Stat(file); err != nil { + return "", fmt.Errorf("Error accessing file %s containing created container name %v", file, err) + } + buf, err := ioutil.ReadFile(file) + if err != nil { + return "", fmt.Errorf("Error reading content of file %s containing created container name %v", file, err) + } + return string(buf), nil +} + // updateOS updates the system OS to the one specified in newConfig func (dn *Daemon) updateOS(config *mcfgv1.MachineConfig) error { if dn.OperatingSystem != MachineConfigDaemonOSRHCOS && dn.OperatingSystem != MachineConfigDaemonOSFCOS { @@ -1305,23 +1286,54 @@ func (dn *Daemon) updateOS(config *mcfgv1.MachineConfig) error { return nil } + if config.Spec.OSImageURL == "" { + return nil + } + newURL := config.Spec.OSImageURL - osMatch, err := compareOSImageURL(dn.bootedOSImageURL, newURL) + if dn.recorder != nil { + dn.recorder.Eventf(getNodeRef(dn.node), corev1.EventTypeNormal, "InClusterUpgrade", fmt.Sprintf("In cluster upgrade to %s", newURL)) + } + + // We need to mount OSContainer in host context so that mounted container is + // accissible by rpm-ostree to update OS and do package layering from coreos-extension repo. + unitName := "mco-mount-container" + glog.Infof("Pulling in image and mounting container on host via systemd-run unit=%s", unitName) + err := exec.Command("systemd-run", "--wait", "--collect", "--unit="+unitName, + "-E", "RPMOSTREE_CLIENT_ID=mco", constants.HostSelfBinary, "mount-container", newURL).Run() if err != nil { + return errors.Wrapf(err, "failed to mount container from image %s", newURL) + } + + var containerName, containerMntLoc string + if containerName, err = ReadFromFile(constants.MountedOSContainerName); err != nil { return err } - if osMatch { - return nil + if containerMntLoc, err = ReadFromFile(constants.MountedOSContainerLocation); err != nil { + return err } - if dn.recorder != nil { - dn.recorder.Eventf(getNodeRef(dn.node), corev1.EventTypeNormal, "InClusterUpgrade", fmt.Sprintf("In cluster upgrade to %s", newURL)) + + // We can remove this check on FCOS when it starts shipping extensions repo + if dn.OperatingSystem == MachineConfigDaemonOSRHCOS { + // Create coreos-extension repo in /etc/yum.repos.d/ . Selinux doesn't allow writing + // content to /etc/ in host context. See https://bugzilla.redhat.com/show_bug.cgi?id=1839065#c23 + if err := addExtRepo(containerMntLoc); err != nil { + return fmt.Errorf("Failed adding extensions repo: %v", err) + } + + defer os.Remove(extensionsRepo) } - glog.Infof("Updating OS to %s", newURL) + defer func() { + exec.Command("podman", "rm", containerName).Run() + exec.Command("podman", "rmi", config.Spec.OSImageURL).Run() + glog.Infof("Deleted container %s and corresponding image %s", containerName, config.Spec.OSImageURL) + }() + // In the cluster case, for now we run indirectly via machine-config-daemon-host.service // for SELinux reasons, see https://bugzilla.redhat.com/show_bug.cgi?id=1839065 if dn.kubeClient != nil { - if err := dn.NodeUpdaterClient.RunPivot(newURL); err != nil { + if err := dn.NodeUpdaterClient.RunPivot(containerName); err != nil { MCDPivotErr.WithLabelValues(newURL, err.Error()).SetToCurrentTime() pivotErr, err2 := ioutil.ReadFile(pivottypes.PivotFailurePath) if err2 != nil || len(pivotErr) == 0 { @@ -1333,7 +1345,7 @@ func (dn *Daemon) updateOS(config *mcfgv1.MachineConfig) error { // If we're here we're invoked via `machine-config-daemon-firstboot.service`, so let's // just run the update directly rather than invoking another service. client := NewNodeUpdaterClient() - _, changed, err := client.PullAndRebase(newURL, false) + changed, err := client.PerformRpmOSTreeOperations(containerName, false) if err != nil { return err }