Merge pull request #12832 from gabrielmougard/feat/improve-vm-live-mi…

…gration-part2

 Instance: Set `migration.stateful=true` be default when creating a new VM

doc/.wordlist.txt

@@ -26,7 +26,7 @@ BPF
 Btrfs
 bugfix
 bugfixes
-Centos
+CentOS
 Ceph
 CephFS
 Ceph's
@@ -303,6 +303,7 @@ VDPA
 VFS
 VFs
 VirtIO
+virtiofs
 virtualize
 virtualized
 VLAN

doc/api-extensions.md

@@ -2348,3 +2348,8 @@ This API extension provides the ability to use flags `--device` when importing a
 ## `instances_uefi_vars`
 
 This API extension indicates that the `/1.0/instances/{name}/uefi-vars` endpoint is supported on the server. This endpoint allows to get the full list of UEFI variables (HTTP method GET) or replace the entire set of UEFI variables (HTTP method PUT).
+
+## `instances_migration_stateful`
+
+This API extension allows newly created VMs to have their `migration.stateful` configuration key automatically set
+through the new server-level configuration key `instances.migration.stateful`. If `migration.stateful` is already set at the profile or instance level then `instances.migration.stateful` is not applied.

doc/config_options.txt

@@ -150,7 +150,7 @@ Using incremental memory transfer of the instance's memory can reduce downtime.
 
 ```{config:option} migration.stateful instance-migration
 :condition: "virtual machine"
-:defaultdesc: "`false`"
+:defaultdesc: "`false` or value from profiles or `instances.migration.stateful` (if set)"
 :liveupdate: "no"
 :shortdesc: "Whether to allow for stateful stop/start and snapshots"
 :type: "bool"
@@ -1657,6 +1657,13 @@ The events can be any combination of `lifecycle`, `logging`, and `ovn`.
 Possible values are `bzip2`, `gzip`, `lzma`, `xz`, or `none`.
 ```
 
+```{config:option} instances.migration.stateful server-miscellaneous
+:scope: "global"
+:shortdesc: "Whether to set `migration.stateful` to `true` for the instances"
+:type: "bool"
+You can override this setting for relevant instances, either in the instance-specific configuration or through a profile.
+```
+
 ```{config:option} instances.nic.host_name server-miscellaneous
 :defaultdesc: "`random`"
 :scope: "global"

doc/howto/move_instances.md

@@ -59,6 +59,14 @@ To do so, ensure the following configuration:
 * Set {config:option}`instance-migration:migration.stateful` to `true` on the instance.
 * Set [`size.state`](devices-disk) of the virtual machine's root disk device to at least the size of the virtual machine's {config:option}`instance-resource-limits:limits.memory` setting.
 
+```{note}
+If you are using a shared storage pool like Ceph RBD to back your instance, you don't need to set [`size.state`](devices-disk) to perform live migration.
+```
+
+```{note}
+When {config:option}`instance-migration:migration.stateful` is enabled in LXD, virtiofs shares are disabled, and files are only shared via the 9P protocol. Consequently, guest OSes lacking 9P support, such as CentOS 8, cannot share files with the host unless stateful migration is disabled. Additionally, the `lxd-agent` will not function for these guests under these conditions.
+```
+
 (live-migration-containers)=
 ### Live migration for containers
 

lxd/api_project.go

@@ -703,7 +703,7 @@ func projectChange(s *state.State, project *api.Project, req api.ProjectPut) res
 
 	// Update the database entry.
 	err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
-		err := projecthelpers.AllowProjectUpdate(tx, project.Name, req.Config, configChanged)
+		err := projecthelpers.AllowProjectUpdate(s.GlobalConfig, tx, project.Name, req.Config, configChanged)
 		if err != nil {
 			return err
 		}
@@ -972,7 +972,7 @@ func projectStateGet(d *Daemon, r *http.Request) response.Response {
 
 	// Get current limits and usage.
 	err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
-		result, err := projecthelpers.GetCurrentAllocations(ctx, tx, name)
+		result, err := projecthelpers.GetCurrentAllocations(s.GlobalConfig.Dump(), ctx, tx, name)
 		if err != nil {
 			return err
 		}

lxd/cluster/config/config.go

@@ -198,6 +198,11 @@ func (c *Config) InstancesPlacementScriptlet() string {
 	return c.m.GetString("instances.placement.scriptlet")
 }
 
+// InstancesMigrationStateful returns the whether or not to auto enable migration.stateful for all VM instances.
+func (c *Config) InstancesMigrationStateful() bool {
+	return c.m.GetBool("instances.migration.stateful")
+}
+
 // LokiServer returns all the Loki settings needed to connect to a server.
 func (c *Config) LokiServer() (apiURL string, authUsername string, authPassword string, apiCACert string, instance string, logLevel string, labels []string, types []string) {
 	if c.m.GetString("loki.types") != "" {
@@ -576,6 +581,14 @@ var ConfigSchema = config.Schema{
 	//  shortdesc: Instance placement scriptlet for automatic instance placement
 	"instances.placement.scriptlet": {Validator: validate.Optional(scriptletLoad.InstancePlacementValidate)},
 
+	// lxdmeta:generate(entities=server; group=miscellaneous; key=instances.migration.stateful)
+	// You can override this setting for relevant instances, either in the instance-specific configuration or through a profile.
+	// ---
+	//  type: bool
+	//  scope: global
+	//  shortdesc: Whether to set `migration.stateful` to `true` for the instances
+	"instances.migration.stateful": {Type: config.Bool, Default: "false"},
+
 	// lxdmeta:generate(entities=server; group=loki; key=loki.auth.username)
 	//
 	// ---

lxd/db/cluster/instances.go

@@ -77,7 +77,7 @@ type InstanceFilter struct {
 }
 
 // ToAPI converts the database Instance to API type.
-func (i *Instance) ToAPI(ctx context.Context, tx *sql.Tx) (*api.Instance, error) {
+func (i *Instance) ToAPI(ctx context.Context, tx *sql.Tx, globalConfig map[string]any) (*api.Instance, error) {
 	profiles, err := GetInstanceProfiles(ctx, tx, i.ID)
 	if err != nil {
 		return nil, err
@@ -108,7 +108,7 @@ func (i *Instance) ToAPI(ctx context.Context, tx *sql.Tx) (*api.Instance, error)
 		return nil, err
 	}
 
-	expandedConfig := instancetype.ExpandInstanceConfig(config, apiProfiles)
+	expandedConfig := instancetype.ExpandInstanceConfig(globalConfig, config, apiProfiles)
 
 	archName, err := osarch.ArchitectureName(i.Architecture)
 	if err != nil {

lxd/db/instances_test.go

@@ -267,7 +267,7 @@ func TestInstanceList(t *testing.T) {
 
 	err = c.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
 		return tx.InstanceList(ctx, func(dbInst db.InstanceArgs, p api.Project) error {
-			dbInst.Config = instancetype.ExpandInstanceConfig(dbInst.Config, dbInst.Profiles)
+			dbInst.Config = instancetype.ExpandInstanceConfig(nil, dbInst.Config, dbInst.Profiles)
 			dbInst.Devices = instancetype.ExpandInstanceDevices(dbInst.Devices, dbInst.Profiles)
 
 			instances = append(instances, dbInst)

lxd/images.go

@@ -1035,7 +1035,7 @@ func imagesPost(d *Daemon, r *http.Request) response.Response {
 	// allowed to use.
 	var budget int64
 	err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
-		budget, err = projectutils.GetImageSpaceBudget(tx, projectName)
+		budget, err = projectutils.GetImageSpaceBudget(s.GlobalConfig, tx, projectName)
 		return err
 	})
 	if err != nil {

lxd/instance/drivers/driver_common.go

@@ -509,7 +509,12 @@ func (d *common) deviceVolatileSetFunc(devName string) func(save map[string]stri
 
 // expandConfig applies the config of each profile in order, followed by the local config.
 func (d *common) expandConfig() error {
-	d.expandedConfig = instancetype.ExpandInstanceConfig(d.localConfig, d.profiles)
+	var globalConfigDump map[string]any
+	if d.state.GlobalConfig != nil {
+		globalConfigDump = d.state.GlobalConfig.Dump()
+	}
+
+	d.expandedConfig = instancetype.ExpandInstanceConfig(globalConfigDump, d.localConfig, d.profiles)
 	d.expandedDevices = instancetype.ExpandInstanceDevices(d.localDevices, d.profiles)
 
 	return nil

lxd/instance/instancetype/instance.go

@@ -912,7 +912,7 @@ var InstanceConfigKeysVM = map[string]func(value string) error{
 	// Enabling this option prevents the use of some features that are incompatible with it.
 	// ---
 	//  type: bool
-	//  defaultdesc: `false`
+	//  defaultdesc: `false` or value from profiles or `instances.migration.stateful` (if set)
 	//  liveupdate: no
 	//  condition: virtual machine
 	//  shortdesc: Whether to allow for stateful stop/start and snapshots

lxd/instance/instancetype/instance_utils.go

@@ -1,14 +1,27 @@
 package instancetype
 
 import (
+	"strconv"
+
 	deviceConfig "github.com/canonical/lxd/lxd/device/config"
 	"github.com/canonical/lxd/shared/api"
 )
 
 // ExpandInstanceConfig expands the given instance config with the config values of the given profiles.
-func ExpandInstanceConfig(config map[string]string, profiles []api.Profile) map[string]string {
+func ExpandInstanceConfig(globalConfig map[string]any, config map[string]string, profiles []api.Profile) map[string]string {
 	expandedConfig := map[string]string{}
 
+	// Apply global config overriding
+	if globalConfig != nil {
+		globalInstancesMigrationStatefulStr, ok := globalConfig["instances.migration.stateful"].(string)
+		if ok {
+			globalInstancesMigrationStateful, _ := strconv.ParseBool(globalInstancesMigrationStatefulStr)
+			if globalInstancesMigrationStateful {
+				expandedConfig["migration.stateful"] = globalInstancesMigrationStatefulStr
+			}
+		}
+	}
+
 	// Apply all the profiles.
 	profileConfigs := make([]map[string]string, len(profiles))
 	for i, profile := range profiles {

lxd/instance_patch.go

@@ -200,7 +200,7 @@ func instancePatch(d *Daemon, r *http.Request) response.Response {
 			apiProfiles = append(apiProfiles, *apiProfile)
 		}
 
-		return projecthelpers.AllowInstanceUpdate(tx, projectName, name, req, c.LocalConfig())
+		return projecthelpers.AllowInstanceUpdate(s.GlobalConfig, tx, projectName, name, req, c.LocalConfig())
 	})
 	if err != nil {
 		return response.SmartError(err)

lxd/instance_put.go

@@ -148,7 +148,7 @@ func instancePut(d *Daemon, r *http.Request) response.Response {
 				apiProfiles = append(apiProfiles, *apiProfile)
 			}
 
-			return projecthelpers.AllowInstanceUpdate(tx, projectName, name, configRaw, inst.LocalConfig())
+			return projecthelpers.AllowInstanceUpdate(s.GlobalConfig, tx, projectName, name, configRaw, inst.LocalConfig())
 		})
 		if err != nil {
 			return response.SmartError(err)

lxd/instances_post.go

@@ -51,7 +51,7 @@ func ensureDownloadedImageFitWithinBudget(s *state.State, r *http.Request, op *o
 
 	var budget int64
 	err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
-		budget, err = project.GetImageSpaceBudget(tx, p.Name)
+		budget, err = project.GetImageSpaceBudget(s.GlobalConfig, tx, p.Name)
 		return err
 	})
 	if err != nil {
@@ -642,7 +642,7 @@ func createFromBackup(s *state.State, r *http.Request, projectName string, data
 			Type:        api.InstanceType(bInfo.Config.Container.Type),
 		}
 
-		return project.AllowInstanceCreation(tx, projectName, req)
+		return project.AllowInstanceCreation(s.GlobalConfig, tx, projectName, req)
 	})
 	if err != nil {
 		return response.SmartError(err)
@@ -1102,7 +1102,7 @@ func instancesPost(d *Daemon, r *http.Request) response.Response {
 		if !clusterNotification {
 			// Check that the project's limits are not violated. Note this check is performed after
 			// automatically generated config values (such as ones from an InstanceType) have been set.
-			err = project.AllowInstanceCreation(tx, targetProjectName, req)
+			err = project.AllowInstanceCreation(s.GlobalConfig, tx, targetProjectName, req)
 			if err != nil {
 				return err
 			}
@@ -1134,7 +1134,12 @@ func instancesPost(d *Daemon, r *http.Request) response.Response {
 				Reason:        apiScriptlet.InstancePlacementReasonNew,
 			}
 
-			reqExpanded.Config = instancetype.ExpandInstanceConfig(reqExpanded.Config, profiles)
+			var globalConfigDump map[string]any
+			if s.GlobalConfig != nil {
+				globalConfigDump = s.GlobalConfig.Dump()
+			}
+
+			reqExpanded.Config = instancetype.ExpandInstanceConfig(globalConfigDump, reqExpanded.Config, profiles)
 			reqExpanded.Devices = instancetype.ExpandInstanceDevices(deviceConfig.NewDevices(reqExpanded.Devices), profiles).CloneNative()
 
 			targetMemberInfo, err = scriptlet.InstancePlacementRun(r.Context(), logger.Log, s, &reqExpanded, candidateMembers, leaderAddress)

lxd/metadata/configuration.json

@@ -176,7 +176,7 @@
 					{
 						"migration.stateful": {
 							"condition": "virtual machine",
-							"defaultdesc": "`false`",
+							"defaultdesc": "`false` or value from profiles or `instances.migration.stateful` (if set)",
 							"liveupdate": "no",
 							"longdesc": "Enabling this option prevents the use of some features that are incompatible with it.",
 							"shortdesc": "Whether to allow for stateful stop/start and snapshots",
@@ -1808,6 +1808,14 @@
 							"type": "string"
 						}
 					},
+					{
+						"instances.migration.stateful": {
+							"longdesc": "You can override this setting for relevant instances, either in the instance-specific configuration or through a profile.",
+							"scope": "global",
+							"shortdesc": "Whether to set `migration.stateful` to `true` for the instances",
+							"type": "bool"
+						}
+					},
 					{
 						"instances.nic.host_name": {
 							"defaultdesc": "`random`",

lxd/network/network_utils_sriov.go

@@ -62,7 +62,12 @@ func SRIOVGetHostDevicesInUse(s *state.State) (map[string]struct{}, error) {
 	err = s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
 		return tx.InstanceList(ctx, func(dbInst db.InstanceArgs, p api.Project) error {
 			// Expand configs so we take into account profile devices.
-			dbInst.Config = instancetype.ExpandInstanceConfig(dbInst.Config, dbInst.Profiles)
+			var globalConfigDump map[string]any
+			if s.GlobalConfig != nil {
+				globalConfigDump = s.GlobalConfig.Dump()
+			}
+
+			dbInst.Config = instancetype.ExpandInstanceConfig(globalConfigDump, dbInst.Config, dbInst.Profiles)
 			dbInst.Devices = instancetype.ExpandInstanceDevices(dbInst.Devices, dbInst.Profiles)
 
 			for name, dev := range dbInst.Devices {

lxd/profiles_utils.go

@@ -17,7 +17,7 @@ import (
 func doProfileUpdate(s *state.State, p api.Project, profileName string, id int64, profile *api.Profile, req api.ProfilePut) error {
 	// Check project limits.
 	err := s.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
-		return project.AllowProfileUpdate(tx, p.Name, profileName, req)
+		return project.AllowProfileUpdate(s.GlobalConfig, tx, p.Name, profileName, req)
 	})
 	if err != nil {
 		return err