Separate flaky and non-flaky

Signed-off-by: apostasie <[email protected]>

Dockerfile

@@ -276,7 +276,8 @@ ARG DEBIAN_FRONTEND=noninteractive
 # `expect` package contains `unbuffer(1)`, which is used for emulating TTY for testing
 RUN apt-get update -qq && apt-get install -qq --no-install-recommends \
  expect \
- git
+ git \
+ make
 COPY --from=goversion /GOVERSION /GOVERSION
 ARG TARGETARCH
 RUN curl -fsSL --proto '=https' --tlsv1.2 https://golang.org/dl/$(cat /GOVERSION).linux-${TARGETARCH:-amd64}.tar.gz | tar xzvC /usr/local
@@ -314,8 +315,6 @@ RUN curl -o nydus-static.tgz -fsSL --proto '=https' --tlsv1.2 "https://github.co
  tar xzf nydus-static.tgz && \
  mv nydus-static/nydus-image nydus-static/nydusd nydus-static/nydusify /usr/bin/ && \
  rm nydus-static.tgz
-CMD ["gotestsum", "--format=testname", "--rerun-fails=2", "--packages=./cmd/nerdctl/...", \
- "--", "-timeout=60m", "-p", "1", "-args", "-test.allow-kill-daemon"]
 
 FROM test-integration AS test-integration-rootless
 # Install SSH for creating systemd user session.
@@ -338,17 +337,10 @@ RUN systemctl disable test-integration-ipfs-offline
 VOLUME /home/rootless/.local/share
 COPY ./Dockerfile.d/test-integration-rootless.sh /
 RUN chmod a+rx /test-integration-rootless.sh
-CMD ["/test-integration-rootless.sh", \
- "gotestsum", "--format=testname", "--rerun-fails=2", "--packages=./cmd/nerdctl/...", \
- "--", "-timeout=60m", "-p", "1", "-args", "-test.allow-kill-daemon"]
 
 # test for CONTAINERD_ROOTLESS_ROOTLESSKIT_PORT_DRIVER=slirp4netns
 FROM test-integration-rootless AS test-integration-rootless-port-slirp4netns
 COPY ./Dockerfile.d/home_rootless_.config_systemd_user_containerd.service.d_port-slirp4netns.conf /home/rootless/.config/systemd/user/containerd.service.d/port-slirp4netns.conf
 RUN chown -R rootless:rootless /home/rootless/.config
 
-FROM test-integration AS test-integration-ipv6
-CMD ["gotestsum", "--format=testname", "--rerun-fails=2", "--packages=./cmd/nerdctl/...", \
- "--", "-timeout=60m", "-p", "1", "-args", "-test.allow-kill-daemon", "-test.only-ipv6"]
-
 FROM base AS demo

Makefile

@@ -86,6 +86,9 @@ lint-yaml:
 lint-shell: $(call recursive_wildcard,$(MAKEFILE_DIR)/,*.sh)
  shellcheck -a -x $^
 
+test-unit:
+ go test -v $(MAKEFILE_DIR)/pkg/...
+
 binaries: nerdctl
 
 install:

hack/test-integration.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+
+# Copyright The containerd Authors.
+
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+
+# http://www.apache.org/licenses/LICENSE-2.0
+
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# shellcheck disable=SC2034,SC2015
+set -o errexit -o errtrace -o functrace -o nounset -o pipefail
+root="$(cd "$(dirname "${BASH_SOURCE[0]:-$PWD}")" 2>/dev/null 1>&2 && pwd)"
+readonly root
+readonly timeout="60m"
+
+# See https://github.com/containerd/nerdctl/blob/main/docs/testing/README.md#about-parallelization
+args=(--format=testname --jsonfile /tmp/test-integration.log --packages="$root"/../cmd/nerdctl/...)
+
+for arg in "$@"; do
+ if [ "$arg" == "-test.only-flaky" ]; then
+ args+=("--rerun-fails=2")
+ break
+ fi
+done
+
+gotestsum "${args[@]}" -- -timeout="$timeout" -p 1 -args -test.allow-kill-daemon "$@"
+
+echo "These are the tests that took more than 20 seconds:"
+gotestsum tool slowest --threshold 20s --jsonfile /tmp/test-integration.log

pkg/testutil/nerdtest/requirements.go

@@ -73,6 +73,11 @@ var OnlyKubernetes = &test.Requirement{
  } else {
  mess = "runner skips Kubernetes compatible tests in the non-Kubernetes environment"
  }
+ _, err := exec.LookPath("kubectl")
+ if err != nil {
+ ret = false
+ mess = fmt.Sprintf("kubectl is not in the path: %+v", err)
+ }
  return ret, mess
  },
 }

pkg/testutil/testutil.go

@@ -787,6 +787,9 @@ func newBase(t *testing.T, ns string, ipv6Compatible bool, kubernetesCompatible
  } else if !base.EnableKubernetes && base.KubernetesCompatible {
  t.Skip("runner skips Kubernetes compatible tests in the non-Kubernetes environment")
  }
+ if !GetFlakyEnvironment() && !GetEnableKubernetes() && !GetEnableIPv6() {
+ t.Skip("legacy tests are considered flaky by default and are skipped unless in the flaky environment")
+ }
  var err error
  switch base.Target {
  case Nerdctl: