Stable 5.0 backports

.deepsource.toml

@@ -17,4 +17,4 @@ name = "go"
 enabled = true
 
   [analyzers.meta]
-  import_paths = ["github.com/lxd/lxd"]
+  import_paths = ["github.com/canonical/lxd"]

.github/CODEOWNERS

@@ -0,0 +1 @@
+* @tomponline

.github/labeler.yml

@@ -4,4 +4,3 @@ API:
 - shared/api/**/*
 Documentation:
 - doc/**/*
-- .sphinx/**/*

.github/workflows/tests.yml

@@ -12,11 +12,16 @@ concurrency:
 
 jobs:
   code-tests:
+    env:
+      CGO_CFLAGS: "-I/home/runner/work/lxd/lxd-test/vendor/raft/include/ -I/home/runner/work/lxd/lxd-test/vendor/dqlite/include/"
+      CGO_LDFLAGS: "-L/home/runner/work/lxd/lxd-test/vendor/raft/.libs/ -L/home/runner/work/lxd/lxd-test/vendor/dqlite/.libs/"
+      LD_LIBRARY_PATH: "/home/runner/work/lxd/lxd-test/vendor/raft/.libs/:/home/runner/work/lxd/lxd-test/vendor/dqlite/.libs/"
+      CGO_LDFLAGS_ALLOW: "(-Wl,-wrap,pthread_create)|(-Wl,-z,now)"
     name: Code tests
     runs-on: ubuntu-22.04
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Dependency Review
         uses: actions/dependency-review-action@v3
@@ -30,35 +35,47 @@ jobs:
       - name: Install dependencies
         run: |
           sudo add-apt-repository ppa:ubuntu-lxc/lxc-git-master -y --no-update
-          sudo add-apt-repository ppa:dqlite/dev -y --no-update
           sudo apt-get update
 
           sudo apt-get install --no-install-recommends -y \
+            build-essential \
             curl \
             gettext \
             git \
             libacl1-dev \
             libcap-dev \
             libdbus-1-dev \
-            libdqlite-dev \
             liblxc-dev \
             lxc-templates \
             libseccomp-dev \
             libselinux-dev \
             libsqlite3-dev \
             libtool \
             libudev-dev \
+            libuv1-dev \
             make \
             pkg-config \
             shellcheck
 
           python3 -m pip install flake8
-          curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.46.2
 
       - name: Download go dependencies
         run: |
           go mod download
 
+      - name: Make LXD tarball and unpack it
+        env:
+          CUSTOM_VERSION: "test"
+        run: |
+          make dist
+          tar -xzf lxd-test.tar.gz -C ~/work/lxd/
+          rm lxd-test.tar.gz
+
+      - name: Build LXD dependencies
+        run: |
+          cd ~/work/lxd/lxd-test
+          make deps
+
       - name: Run LXD build
         run: |
           make
@@ -69,14 +86,15 @@ jobs:
 
       - name: Unit tests (all)
         run: |
-          sudo go test ./...
+          sudo --preserve-env=CGO_CFLAGS,CGO_LDFLAGS,CGO_LDFLAGS_ALLOW,LD_LIBRARY_PATH LD_LIBRARY_PATH=${LD_LIBRARY_PATH} go test ./...
 
   system-tests:
     env:
       CGO_LDFLAGS_ALLOW: "(-Wl,-wrap,pthread_create)|(-Wl,-z,now)"
       LXD_SHIFTFS_DISABLE: "true"
       LXD_CEPH_CLUSTER: "ceph"
       LXD_CEPH_CEPHFS: "cephfs"
+      LXD_CONCURRENT: "1"
       LXD_VERBOSE: "1"
       LXD_OFFLINE: "1"
       LXD_TMPFS: "1"
@@ -104,7 +122,7 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install Go (stable)
         uses: actions/setup-go@v4
@@ -114,7 +132,7 @@ jobs:
 
       - name: Install Go (tip)
         run: |
-          curl -sL https://storage.googleapis.com/go-build-snap/go/linux-amd64/$(git ls-remote https://github.com/golang/go.git HEAD | awk '{print $1;}').tar.gz -o gotip.tar.gz
+          curl -sSfL https://storage.googleapis.com/go-build-snap/go/linux-amd64/$(git ls-remote https://github.com/golang/go.git HEAD | awk '{print $1;}').tar.gz -o gotip.tar.gz
           ls -lah gotip.tar.gz
           mkdir -p ~/sdk/gotip
           tar -C ~/sdk/gotip -xzf gotip.tar.gz
@@ -130,13 +148,16 @@ jobs:
           sudo apt-get update
 
           sudo snap remove lxd --purge
+          sudo snap remove core20 --purge || true
           sudo apt-get autopurge moby-containerd docker uidmap -y
           sudo ip link delete docker0
           sudo nft flush ruleset
 
+          sudo systemctl mask lxc.service
+          sudo systemctl mask lxc-net.service
+
           sudo apt-get install --no-install-recommends -y \
             curl \
-            dnsutils \
             git \
             libacl1-dev \
             libcap-dev \
@@ -156,8 +177,10 @@ jobs:
             btrfs-progs \
             busybox-static \
             dnsmasq-base \
+            easy-rsa \
             gettext \
             jq \
+            lxc-utils \
             lvm2 \
             nftables \
             quota \
@@ -173,6 +196,10 @@ jobs:
             xfsprogs \
             xz-utils \
             zfsutils-linux
+
+          # reclaim some space
+          sudo apt-get clean
+
           mkdir -p "$(go env GOPATH)/bin"
 
       - name: Download go dependencies
@@ -187,7 +214,18 @@ jobs:
         if: ${{ matrix.backend == 'ceph' }}
         run: |
           set -x
-          sudo apt-get install --no-install-recommends -y snapd ceph-common
+
+          # If the rootfs and the ephemeral part are on the same physical disk, giving the whole
+          # disk to microceph would wipe our rootfs. Since it is pretty rare for GitHub Action
+          # runners to have a single disk, we immediately bail rather than trying to gracefully
+          # handle it. Once snapd releases with https://github.com/snapcore/snapd/pull/13150,
+          # we will be able to stop worrying about that special case.
+          if [ "$(stat -c '%d' /)" = "$(stat -c '%d' /mnt)" ]; then
+            echo "FAIL: rootfs and ephemeral part on the same disk, aborting"
+            exit 1
+          fi
+
+          sudo apt-get install --no-install-recommends -y ceph-common
           sudo snap install microceph --edge
           sleep 5
           sudo microceph cluster bootstrap
@@ -199,17 +237,11 @@ jobs:
           for flag in nosnaptrim noscrub nobackfill norebalance norecover noscrub nodeep-scrub; do
               sudo microceph.ceph osd set $flag
           done
-          # Use ephemeral disk mounted on /mnt for ceph OSD.
-          # The block-devices plug doesn't allow accessing /dev/loopX devices so we make those same devices
-          # available under alternate names (/dev/sdiY) that are not used inside GitHub Action runners.
+          # Repurpose the ephemeral disk for ceph OSD.
           sudo swapoff /mnt/swapfile
-          sudo rm -f /mnt/swapfile
-          loop_file="/mnt/ceph-osd.img"
-          sudo fallocate -l 10G "${loop_file}"
-          loop_dev="$(sudo losetup --show --direct-io=on --nooverlap -f "${loop_file}")"
-          devInfo=($(sudo stat -c '%t %T' "${loop_dev}"))
-          sudo mknod -m 0660 /dev/sdia b 0x"${devInfo[0]}" 0x"${devInfo[1]}"
-          sudo microceph disk add --wipe /dev/sdia
+          ephemeral_disk="$(findmnt --noheadings --output SOURCE --target /mnt | sed 's/[0-9]\+$//')"
+          sudo umount /mnt
+          sudo microceph disk add --wipe "${ephemeral_disk}"
           sudo rm -rf /etc/ceph
           sudo ln -s /var/snap/microceph/current/conf/ /etc/ceph
           sudo microceph enable rgw
@@ -226,7 +258,7 @@ jobs:
           chmod +x ~
           echo "root:1000000:1000000000" | sudo tee /etc/subuid /etc/subgid
           cd test
-          sudo --preserve-env=PATH,GOPATH,LXD_VERBOSE,LXD_BACKEND,LXD_CEPH_CLUSTER,LXD_CEPH_CEPHFS,LXD_CEPH_CEPHOBJECT_RADOSGW,LXD_OFFLINE,LXD_SKIP_TESTS,LXD_REQUIRED_TESTS,LXD_SHIFTFS_DISABLE LXD_BACKEND=${{ matrix.backend }} ./main.sh ${{ matrix.suite }}
+          sudo --preserve-env=PATH,GOPATH,GITHUB_ACTIONS,LXD_VERBOSE,LXD_BACKEND,LXD_CEPH_CLUSTER,LXD_CEPH_CEPHFS,LXD_CEPH_CEPHOBJECT_RADOSGW,LXD_OFFLINE,LXD_SKIP_TESTS,LXD_REQUIRED_TESTS,LXD_SHIFTFS_DISABLE LXD_BACKEND=${{ matrix.backend }} ./main.sh ${{ matrix.suite }}
 
   client:
     name: Client tests
@@ -243,7 +275,7 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install Go
         uses: actions/setup-go@v4
@@ -294,7 +326,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install dependencies
         run: |
@@ -313,7 +345,7 @@ jobs:
         uses: get-woke/woke-action@v0
         with:
           fail-on-error: true
-          woke-args: "*.md **/*.md -c https://github.com/canonical-web-and-design/Inclusive-naming/raw/main/config.yml"
+          woke-args: "*.md **/*.md -c https://github.com/canonical/Inclusive-naming/raw/main/config.yml"
 
       - name: Run link checker
         run: |
@@ -339,7 +371,7 @@ jobs:
     if: ${{ github.repository == 'canonical/lxd' && github.event_name == 'push'}}
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup Launchpad SSH access
         env:

.gitignore

@@ -16,10 +16,13 @@ lxd-migrate/lxd-migrate
 lxd-user/lxd-user
 test/devlxd-client/devlxd-client
 test/macaroon-identity/macaroon-identity
+test/syscall/sysinfo/sysinfo
 
 # Sphinx
 doc/html/
+doc/reference/manpages/**/*.md
 doc/.sphinx/deps/
+doc/.sphinx/.doctrees/
 doc/.sphinx/themes/
 doc/.sphinx/venv/
 doc/.sphinx/warnings.txt

CONTRIBUTING.md

@@ -100,8 +100,8 @@ An example of a valid sign-off line is:
 Signed-off-by: Random J Developer <[email protected]>
 ```
 
-Use your real name and a valid e-mail address.
-Sorry, no pseudonyms or anonymous contributions are allowed.
+Use a known identity and a valid e-mail address.
+Sorry, no anonymous contributions are allowed.
 
 We also require each commit be individually signed-off by their author,
 even when part of a larger set. You may find `git commit -s` useful.

Makefile

@@ -3,7 +3,7 @@ POFILES=$(wildcard po/*.po)
 MOFILES=$(patsubst %.po,%.mo,$(POFILES))
 LINGUAS=$(basename $(POFILES))
 POTFILE=po/$(DOMAIN).pot
-VERSION=$(shell grep "var Version" shared/version/flex.go | cut -d'"' -f2)
+VERSION=$(or ${CUSTOM_VERSION},$(shell grep "var Version" shared/version/flex.go | cut -d'"' -f2))
 ARCHIVE=lxd-$(VERSION).tar
 HASH := \#
 TAG_SQLITE3=$(shell printf "$(HASH)include <dqlite.h>\nvoid main(){dqlite_node_id n = 1;}" | $(CC) ${CGO_CFLAGS} -o /dev/null -xc - >/dev/null 2>&1 && echo "libsqlite3")
@@ -112,20 +112,27 @@ ifeq "$(LXD_OFFLINE)" ""
 endif
 	swagger generate spec -o doc/rest-api.yaml -w ./lxd -m
 
+.PHONY: update-metadata
+update-metadata: build
+	@echo "Generating golang documentation metadata"
+	$(GOPATH)/bin/lxd-metadata . --json ./lxd/metadata/configuration.json --txt ./doc/config_options.txt
+
 .PHONY: doc-setup
-doc-setup:
+doc-setup: client
 	@echo "Setting up documentation build environment"
 	python3 -m venv doc/.sphinx/venv
 	. $(SPHINXENV) ; pip install --upgrade -r doc/.sphinx/requirements.txt
+	if [ -d "doc/reference/manpages/" ]; then find doc/reference/manpages/ -name "*.md" -type f -delete; fi
 	rm -Rf doc/html
+	rm -Rf doc/.sphinx/.doctrees
 
 .PHONY: doc
 doc: doc-setup doc-incremental
 
 .PHONY: doc-incremental
 doc-incremental:
 	@echo "Build the documentation"
-	. $(SPHINXENV) ; sphinx-build -c doc/ -b dirhtml doc/ doc/html/ -w doc/.sphinx/warnings.txt
+	. $(SPHINXENV) ; LOCAL_SPHINX_BUILD=True sphinx-build -c doc/ -b dirhtml doc/ doc/html/ -d doc/.sphinx/.doctrees -w doc/.sphinx/warnings.txt
 
 .PHONY: doc-serve
 doc-serve:
@@ -137,12 +144,17 @@ doc-spellcheck: doc
 
 .PHONY: doc-linkcheck
 doc-linkcheck: doc-setup
-	. $(SPHINXENV) ; sphinx-build -c doc/ -b linkcheck doc/ doc/html/
+	. $(SPHINXENV) ; LOCAL_SPHINX_BUILD=True sphinx-build -c doc/ -b linkcheck doc/ doc/html/ -d doc/.sphinx/.doctrees
 
 .PHONY: doc-lint
 doc-lint:
 	doc/.sphinx/.markdownlint/doc-lint.sh
 
+.PHONY: doc-woke
+doc-woke:
+	type woke >/dev/null 2>&1 || { sudo snap install woke; }
+	woke *.md **/*.md -c https://github.com/canonical/Inclusive-naming/raw/main/config.yml
+
 .PHONY: debug
 debug:
 ifeq "$(TAG_SQLITE3)" ""
@@ -228,9 +240,9 @@ po/%.po: po/$(DOMAIN).pot
 
 .PHONY: update-po
 update-po:
+	set -eu; \
 	for lang in $(LINGUAS); do\
-	    msgmerge -U $$lang.po po/$(DOMAIN).pot; \
-	    rm -f $$lang.po~; \
+	    msgmerge --backup=none -U $$lang.po po/$(DOMAIN).pot; \
 	done
 
 .PHONY: update-pot
@@ -245,17 +257,18 @@ build-mo: $(MOFILES)
 
 .PHONY: static-analysis
 static-analysis:
-ifeq ($(shell command -v golangci-lint 2> /dev/null),)
-	go install github.com/golangci/golangci-lint/cmd/[email protected]
+ifeq ($(shell command -v golangci-lint),)
+	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $$(go env GOPATH)/bin
 endif
-ifeq ($(shell command -v shellcheck 2> /dev/null),)
+ifeq ($(shell command -v shellcheck),)
 	echo "Please install shellcheck"
 	exit 1
-endif
+else
 ifneq "$(shell shellcheck --version | grep version: | cut -d ' ' -f2)" "0.8.0"
 	@echo "WARN: shellcheck version is not 0.8.0"
 endif
-ifeq ($(shell command -v flake8 2> /dev/null),)
+endif
+ifeq ($(shell command -v flake8),)
 	echo "Please install flake8"
 	exit 1
 endif
@@ -265,6 +278,5 @@ endif
 	shellcheck test/extras/*.sh
 	run-parts --exit-on-error --regex '.sh' test/lint
 
-.PHONY: tags
-tags: *.go lxd/*.go shared/*.go lxc/*.go
-	find . -type f -name '*.go' | xargs gotags > tags
+tags: */*.go
+	find . -type f -name '*.go' | gotags -L - -f tags