Backports (stable-5.21) (#14297)

.github/workflows/security.yml

@@ -18,7 +18,7 @@ defaults:
 
 jobs:
   trivy-repo:
-    name: Trivy vulnerability scanner - Repository
+    name: Trivy - Repository
     runs-on: ubuntu-22.04
     if: ${{ ( github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' ) && github.ref_name == 'main' && github.repository == 'canonical/lxd' }}
     steps:
@@ -49,7 +49,7 @@ jobs:
           ref: refs/heads/main
 
   trivy-snap:
-    name: Trivy vulnerability scanner - Snap
+    name: Trivy - Snap
     runs-on: ubuntu-22.04
     needs: trivy-repo
     if: ${{ ( github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' ) && github.ref_name == 'main' && github.repository == 'canonical/lxd' }}

.github/workflows/tests.yml

@@ -76,10 +76,8 @@ jobs:
       - name: Install build dependencies
         uses: ./.github/actions/install-lxd-builddeps
 
-      - name: Install test dependencies
+      - name: Download minio/mc
         run: |
-          python3 -m pip install flake8
-
           # Download minio ready to include in dependencies for system tests.
           mkdir -p "$(go env GOPATH)/bin"
           curl -sSfL https://dl.min.io/server/minio/release/linux-amd64/minio --output "$(go env GOPATH)/bin/minio"
@@ -417,13 +415,6 @@ jobs:
           calc: ALL
           tmpdir: /tmp/tics
 
-      # Verify doc links work
-      - name: Run link checker
-        shell: 'script -q -e -c "export TERM=xterm-256color; bash {0}"'
-        run: |
-          set -eux
-          make doc-linkcheck
-
   client:
     name: Client
     strategy:
@@ -554,9 +545,13 @@ jobs:
           fail-on-error: true
           woke-args: "*.md **/*.md -c https://github.com/canonical/Inclusive-naming/raw/main/config.yml"
 
-      # XXX: not running `make doc-linkcheck` here as it often fails due to
-      # remote sites applying rate limits. It is instead ran on a schedule
-      # along with Tiobe TICS
+      - name: Run link checker
+        # Run link checker during scheduled CI runs only
+        if: ${{ github.event_name == 'schedule' }}
+        shell: 'script -q -e -c "export TERM=xterm-256color; bash {0}"'
+        run: |
+          set -eux
+          make doc-linkcheck
 
       - name: Upload documentation artifacts
         if: always()

Makefile

@@ -263,11 +263,6 @@ ifneq "$(shell shellcheck --version | grep version: | cut -d ' ' -f2)" "0.8.0"
 	@echo "WARN: shellcheck version is not 0.8.0"
 endif
 endif
-ifeq ($(shell command -v flake8),)
-	echo "Please install flake8"
-	exit 1
-endif
-	flake8 test/deps/import-busybox
 	shellcheck test/*.sh test/includes/*.sh test/suites/*.sh test/backends/*.sh test/lint/*.sh test/extras/*.sh
 	NOT_EXEC="$(shell find test/lint -type f -not -executable)"; \
 	if [ -n "$$NOT_EXEC" ]; then \

doc/api-extensions.md

@@ -2461,3 +2461,13 @@ When set to `on`, if the host has guest attachment enabled, the guest can reques
 
 This adds entity type metadata to `GET /1.0/metadata/configuration`.
 The entity type metadata is a JSON object under the `entities` key.
+
+## `network_allocations_ovn_uplink`
+
+Includes OVN virtual routers external IPs to `/1.0/network-allocations` responses with the type `uplink`.
+Introduces the `network` field on each allocation, indicating to which network each allocated address belongs.
+And lastly, adds a `project` field on leases, leases can be retrieved via `/1.0/networks/<network>/leases`.
+
+## `network_ovn_uplink_vlan`
+
+Adds support for using a bridge network with a specified VLAN ID as an OVN uplink.

doc/howto/projects_create.md

@@ -158,7 +158,7 @@ For example:
 See [`PUT /1.0/projects/{name}`](swagger:/projects/project_put) for more information.
 ```
 ```{group-tab} UI
-To UI does not currently support editing the full YAML configuration for a project.
+The UI does not currently support editing the full YAML configuration for a project.
 However, you can update several or all configuration options at the same time through the UI.
 ```
 ````

doc/howto/server_expose.md

@@ -25,7 +25,7 @@ The UI requires LXD to be exposed to the network.
 Therefore, you must use the CLI or API to originally expose LXD to the network.
 
 Once you have access to the UI, you can use it to update the setting.
-However, be careful when changing the configured value, because using an invalid value might cause you to loose access to the UI.
+However, be careful when changing the configured value, because using an invalid value might cause you to lose access to the UI.
 ```
 
 Go to {guilabel}`Settings` and edit the value for `core.https_address`.

doc/rest-api.yaml

@@ -3139,6 +3139,10 @@ definitions:
                 description: Whether the entity comes from a network that LXD performs egress source NAT on
                 type: boolean
                 x-go-name: NAT
+            network:
+                description: Network is the name of the network the allocated address belongs to
+                type: string
+                x-go-name: Network
             type:
                 description: Type of the entity consuming the network address
                 type: string
@@ -3289,6 +3293,11 @@ definitions:
                 example: lxd01
                 type: string
                 x-go-name: Location
+            project:
+                description: Name of the project of the entity related to the hostname
+                example: default
+                type: string
+                x-go-name: Project
             type:
                 description: The type of record (static or dynamic)
                 example: dynamic
@@ -11882,7 +11891,7 @@ paths:
                     $ref: '#/responses/Forbidden'
                 "500":
                     $ref: '#/responses/InternalServerError'
-            summary: Get the network allocations in use (`network`, `network-forward` and `load-balancer` and `instance`)
+            summary: Get the network allocations in use (`network`, `network-forward`, `load-balancer`, `uplink` and `instance`)
             tags:
                 - network-allocations
     /1.0/network-zones:

lxc/network_allocations.go

@@ -23,6 +23,7 @@ func (c *cmdNetworkListAllocations) pretty(allocs []api.NetworkAllocations) erro
 	header := []string{
 		i18n.G("USED BY"),
 		i18n.G("ADDRESS"),
+		i18n.G("NETWORK"),
 		i18n.G("TYPE"),
 		i18n.G("NAT"),
 		i18n.G("HARDWARE ADDRESS"),
@@ -33,6 +34,7 @@ func (c *cmdNetworkListAllocations) pretty(allocs []api.NetworkAllocations) erro
 		row := []string{
 			alloc.UsedBy,
 			alloc.Address,
+			alloc.Network,
 			alloc.Type,
 			fmt.Sprint(alloc.NAT),
 			alloc.Hwaddr,

lxd/firewall/drivers/drivers_nftables_templates.go

@@ -32,9 +32,13 @@ chain pstrt{{.chainSeparator}}{{.networkName}} {
 
 	{{- range $ipFamily, $config := .rules}}
 	{{if $config.SNATAddress -}}
-	{{$ipFamily}} saddr {{$config.Subnet}} {{$ipFamily}} daddr != {{$config.Subnet}} snat {{$config.SNATAddress}}
+	# If the output interface name is the network itself the traffic stays within the network.
+	# It's important to check for both the destination address and the output interface
+	# to not falsely snat/masquerade multicast traffic whose destination address it outside of the subnet.
+	# In case br_netfilter is loaded on the host multicast traffic also traverses the postrouting chain.
+	{{$ipFamily}} saddr {{$config.Subnet}} {{$ipFamily}} daddr != {{$config.Subnet}} oifname != {{$.networkName}} snat {{$config.SNATAddress}}
 	{{else -}}
-	{{$ipFamily}} saddr {{$config.Subnet}} {{$ipFamily}} daddr != {{$config.Subnet}} masquerade
+	{{$ipFamily}} saddr {{$config.Subnet}} {{$ipFamily}} daddr != {{$config.Subnet}} oifname != {{$.networkName}} masquerade
 	{{- end}}
 	{{- end}}
 }

lxd/firewall/drivers/drivers_xtables.go

@@ -352,6 +352,11 @@ func (d Xtables) networkSetupOutboundNAT(networkName string, subnet *net.IPNet,
 	args := []string{
 		"-s", subnet.String(),
 		"!", "-d", subnet.String(),
+		// If the output interface name is the network itself the traffic stays within the network.
+		// It's important to check for both the destination address and the output interface
+		// to not falsely snat/masquerade multicast traffic whose destination address it outside of the subnet.
+		// In case br_netfilter is loaded on the host multicast traffic also traverses the postrouting chain.
+		"!", "-o", networkName,
 	}
 
 	// If SNAT IP not supplied then use the IP of the outbound interface (MASQUERADE).

lxd/network/driver_bridge.go

@@ -1506,7 +1506,7 @@ func (n *bridge) setup(oldConfig map[string]string) error {
 
 		subnetSize, _ := subnet.Mask.Size()
 
-		if subnetSize > 64 {
+		if subnetSize < 64 {
 			n.logger.Warn("IPv6 networks with a prefix larger than 64 aren't properly supported by dnsmasq")
 			err = n.state.DB.Cluster.Transaction(context.TODO(), func(ctx context.Context, tx *db.ClusterTx) error {
 				return tx.UpsertWarningLocalNode(ctx, n.project, entity.TypeNetwork, int(n.id), warningtype.LargerIPv6PrefixThanSupported, "")
@@ -3440,15 +3440,17 @@ func (n *bridge) forwardSetupFirewall() error {
 
 // Leases returns a list of leases for the bridged network. It will reach out to other cluster members as needed.
 // The projectName passed here refers to the initial project from the API request which may differ from the network's project.
+// If projectName is empty, get leases from all projects.
 func (n *bridge) Leases(projectName string, clientType request.ClientType) ([]api.NetworkLease, error) {
 	var err error
 	var projectMacs []string
+	instanceProjects := make(map[string]string)
 	leases := []api.NetworkLease{}
 
 	// Get all static leases.
 	if clientType == request.ClientTypeNormal {
 		// If requested project matches network's project then include gateway and downstream uplink IPs.
-		if projectName == n.project {
+		if projectName == n.project || projectName == "" {
 			// Add our own gateway IPs.
 			for _, addr := range []string{n.config["ipv4.address"], n.config["ipv6.address"]} {
 				ip, _, _ := net.ParseCIDR(addr)
@@ -3486,6 +3488,7 @@ func (n *bridge) Leases(projectName string, clientType request.ClientType) ([]ap
 								Hostname: fmt.Sprintf("%s-%s.uplink", projectName, network.Name),
 								Address:  v,
 								Type:     "uplink",
+								Project:  projectName,
 							})
 						}
 					}
@@ -3494,13 +3497,20 @@ func (n *bridge) Leases(projectName string, clientType request.ClientType) ([]ap
 		}
 
 		// Get all the instances in the requested project that are connected to this network.
-		filter := dbCluster.InstanceFilter{Project: &projectName}
+		var filter dbCluster.InstanceFilter
+		if projectName != "" {
+			filter = dbCluster.InstanceFilter{Project: &projectName}
+		}
+
 		err = UsedByInstanceDevices(n.state, n.Project(), n.Name(), n.Type(), func(inst db.InstanceArgs, nicName string, nicConfig map[string]string) error {
 			// Fill in the hwaddr from volatile.
 			if nicConfig["hwaddr"] == "" {
 				nicConfig["hwaddr"] = inst.Config[fmt.Sprintf("volatile.%s.hwaddr", nicName)]
 			}
 
+			// Keep instance project to use on dynamic leases.
+			instanceProjects[inst.Name] = inst.Project
+
 			// Record the MAC.
 			hwAddr, _ := net.ParseMAC(nicConfig["hwaddr"])
 			if hwAddr != nil {
@@ -3516,6 +3526,7 @@ func (n *bridge) Leases(projectName string, clientType request.ClientType) ([]ap
 					Hwaddr:   hwAddr.String(),
 					Type:     "static",
 					Location: inst.Node,
+					Project:  inst.Project,
 				})
 			}
 
@@ -3527,6 +3538,7 @@ func (n *bridge) Leases(projectName string, clientType request.ClientType) ([]ap
 					Hwaddr:   hwAddr.String(),
 					Type:     "static",
 					Location: inst.Node,
+					Project:  inst.Project,
 				})
 			}
 
@@ -3541,6 +3553,7 @@ func (n *bridge) Leases(projectName string, clientType request.ClientType) ([]ap
 						Hwaddr:   hwAddr.String(),
 						Type:     "dynamic",
 						Location: inst.Node,
+						Project:  inst.Project,
 					})
 				}
 			}
@@ -3607,6 +3620,7 @@ func (n *bridge) Leases(projectName string, clientType request.ClientType) ([]ap
 				Hwaddr:   macStr,
 				Type:     "dynamic",
 				Location: n.state.ServerName,
+				Project:  instanceProjects[fields[3]],
 			})
 		}
 	}

lxd/network/driver_ovn.go

@@ -596,13 +596,16 @@ func (n *ovn) Validate(config map[string]string) error {
 		return err
 	}
 
-	// Check that if IPv6 enabled then the network size must be at least a /64 as both RA and DHCPv6
-	// in OVN (as it generates addresses using EUI64) require at least a /64 subnet to operate.
-	_, ipv6Net, _ := net.ParseCIDR(config["ipv6.address"])
-	if ipv6Net != nil {
-		ones, _ := ipv6Net.Mask.Size()
-		if ones < 64 {
-			return fmt.Errorf("IPv6 subnet must be at least a /64")
+	// Check that if stateless DHCPv6 is enabled and IPv6 subnet is set then the network size
+	// must be at least a /64 as both RA and DHCPv6 in OVN (as it generates addresses using EUI64)
+	// require at least a /64 subnet to operate.
+	if shared.IsTrueOrEmpty(config["ipv6.dhcp"]) && shared.IsFalseOrEmpty(config["ipv6.dhcp.stateful"]) {
+		_, ipv6Net, _ := net.ParseCIDR(config["ipv6.address"])
+		if ipv6Net != nil {
+			ones, _ := ipv6Net.Mask.Size()
+			if ones > 64 {
+				return fmt.Errorf("IPv6 subnet must be at least a /64 when stateless DHCPv6 is enabled")
+			}
 		}
 	}
 
@@ -1531,6 +1534,14 @@ func (n *ovn) startUplinkPortBridgeNative(uplinkNet Network, bridgeDevice string
 		return fmt.Errorf("Failed to bring up uplink veth interface %q: %w", vars.uplinkEnd, err)
 	}
 
+	// Add VLAN filter entry to the uplink end of the veth interface.
+	if uplinkNetConfig["vlan"] != "" {
+		err = link.BridgeVLANAdd(uplinkNetConfig["vlan"], true, true, false)
+		if err != nil {
+			return fmt.Errorf("Failed to configure VLAN for uplink veth interface %q: %w", vars.uplinkEnd, err)
+		}
+	}
+
 	// Ensure uplink OVS end veth interface is up.
 	link = &ip.Link{Name: vars.ovsEnd}
 	err = link.SetUp()
@@ -1646,6 +1657,11 @@ func (n *ovn) startUplinkPortPhysical(uplinkNet Network) error {
 		return n.startUplinkPortBridgeNative(uplinkNet, uplinkHostName)
 	}
 
+	// Handle case where uplink interface is bridge and VLAN is specified.
+	if IsNativeBridge(uplinkConfig["parent"]) && uplinkConfig["vlan"] != "" {
+		return n.startUplinkPortBridgeNative(uplinkNet, uplinkConfig["parent"])
+	}
+
 	// Detect if uplink interface is a OVS bridge.
 	ovs := openvswitch.NewOVS()
 	isOVSBridge, _ := ovs.BridgeExists(uplinkHostName)
@@ -1853,7 +1869,7 @@ func (n *ovn) deleteUplinkPortPhysical(uplinkNet Network) error {
 	uplinkHostName := GetHostDevice(uplinkConfig["parent"], uplinkConfig["vlan"])
 
 	// Detect if uplink interface is a native bridge.
-	if IsNativeBridge(uplinkHostName) {
+	if IsNativeBridge(uplinkHostName) || (IsNativeBridge(uplinkConfig["parent"]) && uplinkConfig["vlan"] != "") {
 		return n.deleteUplinkPortBridgeNative(uplinkNet)
 	}
 
@@ -4411,7 +4427,7 @@ func (n *ovn) DHCPv6Subnet() *net.IPNet {
 
 	if subnet != nil {
 		ones, _ := subnet.Mask.Size()
-		if ones < 64 {
+		if ones > 64 {
 			return nil // OVN only supports DHCPv6 allocated using EUI64 (which needs at least a /64).
 		}
 	}
@@ -5421,12 +5437,13 @@ func (n *ovn) LoadBalancerDelete(listenAddress string, clientType request.Client
 }
 
 // Leases returns a list of leases for the OVN network. Those are directly extracted from the OVN database.
+// If projectName is empty, get leases from all projects.
 func (n *ovn) Leases(projectName string, clientType request.ClientType) ([]api.NetworkLease, error) {
 	var err error
 	leases := []api.NetworkLease{}
 
 	// If requested project matches network's project then include gateway IPs.
-	if projectName == n.project {
+	if projectName == n.project || projectName == "" {
 		// Add our own gateway IPs.
 		for _, addr := range []string{n.config["ipv4.address"], n.config["ipv6.address"]} {
 			ip, _, _ := net.ParseCIDR(addr)
@@ -5435,13 +5452,18 @@ func (n *ovn) Leases(projectName string, clientType request.ClientType) ([]api.N
 					Hostname: fmt.Sprintf("%s.gw", n.Name()),
 					Address:  ip.String(),
 					Type:     "gateway",
+					Project:  n.project,
 				})
 			}
 		}
 	}
 
 	// Get all the instances in the requested project that are connected to this network.
-	filter := dbCluster.InstanceFilter{Project: &projectName}
+	var filter dbCluster.InstanceFilter
+	if projectName != "" {
+		filter = dbCluster.InstanceFilter{Project: &projectName}
+	}
+
 	err = UsedByInstanceDevices(n.state, n.Project(), n.Name(), n.Type(), func(inst db.InstanceArgs, nicName string, nicConfig map[string]string) error {
 		// Get the instance UUID needed for OVN port name generation.
 		instanceUUID := inst.Config["volatile.uuid"]
@@ -5475,6 +5497,7 @@ func (n *ovn) Leases(projectName string, clientType request.ClientType) ([]api.N
 				Hwaddr:   hwAddr.String(),
 				Type:     leaseType,
 				Location: inst.Node,
+				Project:  inst.Project,
 			})
 		}