Merge pull request #3418 from apostasie/dev-3130-tesutil-nextgen

New test tooling

.github/workflows/test-canary.yml

@@ -112,5 +112,5 @@ jobs:
  ctrdVersion: ${{ env.CONTAINERD_VERSION }}
  run: powershell hack/configure-windows-ci.ps1
  - name: "Run integration tests"
- # See https://github.com/containerd/nerdctl/blob/main/docs/dev/testing.md#about-parallelization
+ # See https://github.com/containerd/nerdctl/blob/main/docs/testing/README.md#about-parallelization
  run: go test -p 1 -v ./cmd/nerdctl/...

.github/workflows/test-kube.yml

@@ -22,7 +22,7 @@ jobs:
  with:
  fetch-depth: 1
  - name: "Run Kubernetes integration tests"
- # See https://github.com/containerd/nerdctl/blob/main/docs/dev/testing.md#about-parallelization
+ # See https://github.com/containerd/nerdctl/blob/main/docs/testing/README.md#about-parallelization
  run: |
  ./hack/build-integration-kubernetes.sh
  sudo ./_output/nerdctl exec nerdctl-test-control-plane bash -c -- 'export TMPDIR="$HOME"/tmp; mkdir -p "$TMPDIR"; cd /nerdctl-source; /usr/local/go/bin/go test -p 1 ./cmd/nerdctl/... -test.only-kubernetes'

.github/workflows/test.yml

@@ -290,15 +290,15 @@ jobs:
  timeout_minutes: 30
  max_attempts: 2
  retry_on: error
- # See https://github.com/containerd/nerdctl/blob/main/docs/dev/testing.md#about-parallelization
+ # See https://github.com/containerd/nerdctl/blob/main/docs/testing/README.md#about-parallelization
  command: go test -p 1 -timeout 20m -v -exec sudo ./cmd/nerdctl/... -args -test.target=docker -test.allow-kill-daemon
  - name: "Ensure that the IPv6 integration test suite is compatible with Docker"
  uses: nick-fields/retry@v3
  with:
  timeout_minutes: 30
  max_attempts: 2
  retry_on: error
- # See https://github.com/containerd/nerdctl/blob/main/docs/dev/testing.md#about-parallelization
+ # See https://github.com/containerd/nerdctl/blob/main/docs/testing/README.md#about-parallelization
  command: go test -p 1 -timeout 20m -v -exec sudo ./cmd/nerdctl/... -args -test.target=docker -test.allow-kill-daemon -test.only-ipv6
 
  test-integration-windows:
@@ -332,7 +332,7 @@ jobs:
  run: powershell hack/configure-windows-ci.ps1
  # TODO: Run unit tests
  - name: "Run integration tests"
- # See https://github.com/containerd/nerdctl/blob/main/docs/dev/testing.md#about-parallelization
+ # See https://github.com/containerd/nerdctl/blob/main/docs/testing/README.md#about-parallelization
  run: go test -p 1 -v ./cmd/nerdctl/...
 
  test-integration-freebsd:

README.md

@@ -253,7 +253,7 @@ Using `go install github.com/containerd/nerdctl/v2/cmd/nerdctl` is possible, but
 
 ### Testing
 
-See [testing nerdctl](docs/dev/testing.md).
+See [testing nerdctl](docs/testing/README.md).
 
 ### Contributing to nerdctl
 

cmd/nerdctl/main_linux_test.go

@@ -20,17 +20,39 @@ import (
  "testing"
 
  "github.com/containerd/nerdctl/v2/pkg/testutil"
+ "github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest"
+ "github.com/containerd/nerdctl/v2/pkg/testutil/test"
 )
 
 // TestIssue108 tests https://github.com/containerd/nerdctl/issues/108
 // ("`nerdctl run --net=host -it` fails while `nerdctl run -it --net=host` works")
 func TestIssue108(t *testing.T) {
- base := testutil.NewBase(t)
- // unbuffer(1) emulates tty, which is required by `nerdctl run -t`.
- // unbuffer(1) can be installed with `apt-get install expect`.
- unbuffer := []string{"unbuffer"}
- base.CmdWithHelper(unbuffer, "run", "-it", "--rm", "--net=host", testutil.AlpineImage,
- "echo", "this was always working").AssertOK()
- base.CmdWithHelper(unbuffer, "run", "--rm", "--net=host", "-it", testutil.AlpineImage,
- "echo", "this was not working due to issue #108").AssertOK()
+ nerdtest.Setup()
+
+ testGroup := &test.Group{
+ {
+ Description: "-it --net=host",
+ Require: test.Binary("unbuffer"),
+ Command: func(data test.Data, helpers test.Helpers) test.Command {
+ return helpers.
+ Command("run", "-it", "--rm", "--net=host", testutil.AlpineImage, "echo", "this was always working").
+ WithWrapper("unbuffer")
+ },
+ // Note: unbuffer will merge stdout and stderr, preventing exact match here
+ Expected: test.Expects(0, nil, test.Contains("this was always working")),
+ },
+ {
+ Description: "--net=host -it",
+ Require: test.Binary("unbuffer"),
+ Command: func(data test.Data, helpers test.Helpers) test.Command {
+ return helpers.
+ Command("run", "--rm", "--net=host", "-it", testutil.AlpineImage, "echo", "this was not working due to issue #108").
+ WithWrapper("unbuffer")
+ },
+ // Note: unbuffer will merge stdout and stderr, preventing exact match here
+ Expected: test.Expects(0, nil, test.Contains("this was not working due to issue #108")),
+ },
+ }
+
+ testGroup.Run(t)
 }

cmd/nerdctl/main_test.go

@@ -17,15 +17,14 @@
 package main
 
 import (
- "os"
- "path/filepath"
+ "errors"
  "testing"
 
- "gotest.tools/v3/assert"
-
  "github.com/containerd/containerd/v2/defaults"
 
  "github.com/containerd/nerdctl/v2/pkg/testutil"
+ "github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest"
+ "github.com/containerd/nerdctl/v2/pkg/testutil/test"
 )
 
 func TestMain(m *testing.M) {
@@ -34,56 +33,100 @@ func TestMain(m *testing.M) {
 
 // TestUnknownCommand tests https://github.com/containerd/nerdctl/issues/487
 func TestUnknownCommand(t *testing.T) {
- t.Parallel()
- base := testutil.NewBase(t)
- base.Cmd("non-existent-command").AssertFail()
- base.Cmd("non-existent-command", "info").AssertFail()
- base.Cmd("system", "non-existent-command").AssertFail()
- base.Cmd("system", "non-existent-command", "info").AssertFail()
- base.Cmd("system").AssertOK() // show help without error
- base.Cmd("system", "info").AssertOutContains("Kernel Version:")
- base.Cmd("info").AssertOutContains("Kernel Version:")
+ nerdtest.Setup()
+
+ var unknownSubCommand = errors.New("unknown subcommand")
+
+ testGroup := &test.Group{
+ {
+ Description: "non-existent-command",
+ Command: test.RunCommand("non-existent-command"),
+ Expected: test.Expects(1, []error{unknownSubCommand}, nil),
+ },
+ {
+ Description: "non-existent-command info",
+ Command: test.RunCommand("non-existent-command", "info"),
+ Expected: test.Expects(1, []error{unknownSubCommand}, nil),
+ },
+ {
+ Description: "system non-existent-command",
+ Command: test.RunCommand("system", "non-existent-command"),
+ Expected: test.Expects(1, []error{unknownSubCommand}, nil),
+ },
+ {
+ Description: "system non-existent-command info",
+ Command: test.RunCommand("system", "non-existent-command", "info"),
+ Expected: test.Expects(1, []error{unknownSubCommand}, nil),
+ },
+ {
+ Description: "system",
+ Command: test.RunCommand("system"),
+ Expected: test.Expects(0, nil, nil),
+ },
+ {
+ Description: "system info",
+ Command: test.RunCommand("system", "info"),
+ Expected: test.Expects(0, nil, test.Contains("Kernel Version:")),
+ },
+ {
+ Description: "info",
+ Command: test.RunCommand("info"),
+ Expected: test.Expects(0, nil, test.Contains("Kernel Version:")),
+ },
+ }
+
+ testGroup.Run(t)
 }
 
-// TestNerdctlConfig validates the configuration precedence [CLI, Env, TOML, Default].
+// TestNerdctlConfig validates the configuration precedence [CLI, Env, TOML, Default] and broken config rejection
 func TestNerdctlConfig(t *testing.T) {
- testutil.DockerIncompatible(t)
- t.Parallel()
- tomlPath := filepath.Join(t.TempDir(), "nerdctl.toml")
- err := os.WriteFile(tomlPath, []byte(`
-snapshotter = "dummy-snapshotter-via-toml"
-`), 0400)
- assert.NilError(t, err)
- base := testutil.NewBase(t)
-
- // [Default]
- base.Cmd("info", "-f", "{{.Driver}}").AssertOutExactly(defaults.DefaultSnapshotter + "\n")
-
- // [TOML, Default]
- base.Env = append(base.Env, "NERDCTL_TOML="+tomlPath)
- base.Cmd("info", "-f", "{{.Driver}}").AssertOutExactly("dummy-snapshotter-via-toml\n")
-
- // [CLI, TOML, Default]
- base.Cmd("info", "-f", "{{.Driver}}", "--snapshotter=dummy-snapshotter-via-cli").AssertOutExactly("dummy-snapshotter-via-cli\n")
-
- // [Env, TOML, Default]
- base.Env = append(base.Env, "CONTAINERD_SNAPSHOTTER=dummy-snapshotter-via-env")
- base.Cmd("info", "-f", "{{.Driver}}").AssertOutExactly("dummy-snapshotter-via-env\n")
-
- // [CLI, Env, TOML, Default]
- base.Cmd("info", "-f", "{{.Driver}}", "--snapshotter=dummy-snapshotter-via-cli").AssertOutExactly("dummy-snapshotter-via-cli\n")
-}
-
-func TestNerdctlConfigBad(t *testing.T) {
- testutil.DockerIncompatible(t)
- t.Parallel()
- tomlPath := filepath.Join(t.TempDir(), "config.toml")
- err := os.WriteFile(tomlPath, []byte(`
-# containerd config, not nerdctl config
-version = 2
-`), 0400)
- assert.NilError(t, err)
- base := testutil.NewBase(t)
- base.Env = append(base.Env, "NERDCTL_TOML="+tomlPath)
- base.Cmd("info").AssertFail()
+ nerdtest.Setup()
+
+ tc := &test.Case{
+ Description: "Nerdctl configuration",
+ // Docker does not support nerdctl.toml obviously
+ Require: test.Not(nerdtest.Docker),
+ SubTests: []*test.Case{
+ {
+ Description: "Default",
+ Command: test.RunCommand("info", "-f", "{{.Driver}}"),
+ Expected: test.Expects(0, nil, test.Equals(defaults.DefaultSnapshotter+"\n")),
+ },
+ {
+ Description: "TOML > Default",
+ Command: test.RunCommand("info", "-f", "{{.Driver}}"),
+ Expected: test.Expects(0, nil, test.Equals("dummy-snapshotter-via-toml\n")),
+ Data: test.WithConfig(nerdtest.NerdctlToml, `snapshotter = "dummy-snapshotter-via-toml"`),
+ },
+ {
+ Description: "Cli > TOML > Default",
+ Command: test.RunCommand("info", "-f", "{{.Driver}}", "--snapshotter=dummy-snapshotter-via-cli"),
+ Expected: test.Expects(0, nil, test.Equals("dummy-snapshotter-via-cli\n")),
+ Data: test.WithConfig(nerdtest.NerdctlToml, `snapshotter = "dummy-snapshotter-via-toml"`),
+ },
+ {
+ Description: "Env > TOML > Default",
+ Command: test.RunCommand("info", "-f", "{{.Driver}}"),
+ Env: map[string]string{"CONTAINERD_SNAPSHOTTER": "dummy-snapshotter-via-env"},
+ Expected: test.Expects(0, nil, test.Equals("dummy-snapshotter-via-env\n")),
+ Data: test.WithConfig(nerdtest.NerdctlToml, `snapshotter = "dummy-snapshotter-via-toml"`),
+ },
+ {
+ Description: "Cli > Env > TOML > Default",
+ Command: test.RunCommand("info", "-f", "{{.Driver}}", "--snapshotter=dummy-snapshotter-via-cli"),
+ Env: map[string]string{"CONTAINERD_SNAPSHOTTER": "dummy-snapshotter-via-env"},
+ Expected: test.Expects(0, nil, test.Equals("dummy-snapshotter-via-cli\n")),
+ Data: test.WithConfig(nerdtest.NerdctlToml, `snapshotter = "dummy-snapshotter-via-toml"`),
+ },
+ {
+ Description: "Broken config",
+ Command: test.RunCommand("info"),
+ Expected: test.Expects(1, []error{errors.New("failed to load nerdctl config")}, nil),
+ Data: test.WithConfig(nerdtest.NerdctlToml, `# containerd config, not nerdctl config
+version = 2`),
+ },
+ },
+ }
+
+ tc.Run(t)
 }