WIP: system test parallelization: two-pass approach

.cirrus.yml

@@ -229,6 +229,7 @@ alt_build_task:
  only_if: &no_rhel_release |
  $CIRRUS_BRANCH !=~ 'v[0-9\.]+-rhel' &&
  $CIRRUS_BASE_BRANCH !=~ 'v[0-9\.]+-rhel'
+ skip: $CI == $CI
  env:
  <<: *stdenvars
  TEST_FLAVOR: "altbuild"
@@ -266,6 +267,7 @@ osx_alt_build_task:
  alias: osx_alt_build
  # Docs: ./contrib/cirrus/CIModes.md
  only_if: *no_rhel_release # RHEL never releases podman mac installer binary
+ skip: $CI == $CI
  persistent_worker: &mac_pw
  labels:
  os: darwin
@@ -444,6 +446,7 @@ win_installer_task:
  CONTAINERS_MACHINE_PROVIDER: 'hyperv'
  alias: win_installer
  only_if: *no_rhel_release
+ skip: $CI == $CI
  depends_on: *build
  ec2_instance: &windows
  image: "${WINDOWS_AMI}"
@@ -622,6 +625,7 @@ local_integration_test_task: &local_integration_test_task
  changesInclude('.cirrus.yml', 'Makefile', 'contrib/cirrus/**', 'vendor/**', 'test/tools/**', 'test/registries*.conf', 'hack/**', 'version/rawversion/*') ||
  changesInclude('test/e2e/**', 'test/utils/**') ||
  (changesInclude('**/*.go', '**/*.c', '**/*.h') && !changesIncludeOnly('test/**', 'pkg/machine/e2e/**'))
+ skip: $CI == $CI
  depends_on: *build
  matrix: *platform_axis
  # integration tests scale well with cpu as they are parallelized
@@ -647,6 +651,7 @@ local_integration_test_task: &local_integration_test_task
 remote_integration_test_task:
  <<: *local_integration_test_task
  alias: remote_integration_test
+ skip: $CI == $CI
  env:
  TEST_FLAVOR: int
  PODBIN_NAME: remote
@@ -659,6 +664,7 @@ container_integration_test_task:
  alias: container_integration_test
  # Docs: ./contrib/cirrus/CIModes.md
  only_if: *only_if_int_test
+ skip: $CI == $CI
  depends_on: *build
  gce_instance: *fastvm
  env:
@@ -677,6 +683,7 @@ rootless_integration_test_task:
  alias: rootless_integration_test
  # Docs: ./contrib/cirrus/CIModes.md
  only_if: *only_if_int_test
+ skip: $CI == $CI
  depends_on: *build
  matrix: *platform_axis
  gce_instance: *fastvm
@@ -700,6 +707,7 @@ podman_machine_task:
  $CIRRUS_CHANGE_TITLE =~ '.*CI:ALL.*' ||
  changesInclude('.cirrus.yml', 'Makefile', 'contrib/cirrus/**', 'vendor/**', 'test/tools/**', 'test/registries*.conf', 'hack/**', 'version/rawversion/*') ||
  changesInclude('cmd/podman/machine/**', 'pkg/machine/**', '**/*machine*.go')
+ skip: $CI == $CI
  depends_on: *build
  ec2_instance:
  image: "${VM_IMAGE_NAME}"
@@ -722,6 +730,7 @@ podman_machine_aarch64_task:
  name: *std_name_fmt
  alias: podman_machine_aarch64
  only_if: *only_if_machine_test
+ skip: $CI == $CI
  depends_on: *build
  ec2_instance:
  <<: *standard_build_ec2_aarch64
@@ -752,9 +761,10 @@ podman_machine_windows_task:
  # everywhere to do so here it would mean we would need duplicate the
  # full big only_if condition which is more difficult to maintain so
  # use the skip here.
- skip: &skip_rhel_release |
- $CIRRUS_BRANCH =~ 'v[0-9\.]+-rhel' ||
- $CIRRUS_BASE_BRANCH =~ 'v[0-9\.]+-rhel'
+# skip: &skip_rhel_release |
+# $CIRRUS_BRANCH =~ 'v[0-9\.]+-rhel' ||
+# $CIRRUS_BASE_BRANCH =~ 'v[0-9\.]+-rhel'
+ skip: $CI == $CI
  depends_on: *build
  ec2_instance:
  <<: *windows
@@ -780,7 +790,8 @@ podman_machine_mac_task:
  name: *std_name_fmt
  alias: podman_machine_mac
  only_if: *only_if_machine_test
- skip: *skip_rhel_release
+# skip: *skip_rhel_release
+ skip: $CI == $CI
  depends_on: *build
  persistent_worker: *mac_pw
  timeout_in: 35m
@@ -955,6 +966,7 @@ buildah_bud_test_task:
  $CIRRUS_CHANGE_TITLE =~ '.*CI:ALL.*' ||
  changesInclude('.cirrus.yml', 'Makefile', 'contrib/cirrus/**', 'vendor/**', 'test/tools/**', 'test/registries*.conf', 'hack/**', 'version/rawversion/*') ||
  changesInclude('**/*build*.go', 'test/buildah-bud/**')
+ skip: $CI == $CI
  depends_on: *build
  env:
  <<: *stdenvars
@@ -984,6 +996,7 @@ upgrade_test_task:
  changesInclude('.cirrus.yml', 'Makefile', 'contrib/cirrus/**', 'vendor/**', 'test/tools/**', 'test/registries*.conf', 'hack/**', 'version/rawversion/*') ||
  changesInclude('test/upgrade/**', 'test/system/*.bash') ||
  (changesInclude('**/*.go', '**/*.c', '**/*.h') && !changesIncludeOnly('test/**', 'pkg/machine/e2e/**'))
+ skip: $CI == $CI
  depends_on: *build
  matrix:
  - env:

go.mod

@@ -226,3 +226,5 @@ require (
  gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
  tags.cncf.io/container-device-interface/specs-go v0.8.0 // indirect
 )
+
+replace github.com/nxadm/tail => github.com/Luap99/tail v0.0.0-20240626140224-ad4e60e8be25

go.sum

@@ -10,6 +10,8 @@ github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg6
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0=
 github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
+github.com/Luap99/tail v0.0.0-20240626140224-ad4e60e8be25 h1:fz7HD7A+DFIBortMJp4kCr0WqU5FXjQHPkXPMTHOsrw=
+github.com/Luap99/tail v0.0.0-20240626140224-ad4e60e8be25/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc=
 github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
 github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
 github.com/Microsoft/hcsshim v0.12.6 h1:qEnZjoHXv+4/s0LmKZWE0/AiZmMWEIkFfWBSf1a0wlU=
@@ -384,8 +386,6 @@ github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
 github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
 github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
-github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY=
-github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc=
 github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=
 github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
 github.com/onsi/ginkgo/v2 v2.20.2 h1:7NVCeyIWROIAheY21RLS+3j2bb52W0W82tkberYytp4=

hack/bats

@@ -131,6 +131,9 @@ export PODMAN_ROOTLESS_USER=$(id -un)
 # Make sure to always check for leaks when running locally
 export PODMAN_BATS_LEAK_CHECK=1
 
+# FIXME
+export BATS_LOGDIR=$(mktemp -d --tmpdir podman-bats-logs.XXXXXXXXX)
+
 # Root
 if [[ "$TEST_ROOT" ]]; then
  echo "# bats ${bats_opts[*]} ${bats_filter[*]} $TESTS"
@@ -140,15 +143,18 @@ if [[ "$TEST_ROOT" ]]; then
  --preserve-env=OCI_RUNTIME \
  --preserve-env=CONTAINERS_HELPER_BINARY_DIR \
  --preserve-env=PODMAN_ROOTLESS_USER \
- bats "${bats_opts[@]}" "${bats_filter[@]}" $TESTS
+ --preserve-env=BATS_LOGDIR \
+ bats "${bats_opts[@]}" "${bats_filter[@]}" \
+ --gather-test-outputs-in "$BATS_LOGDIR/root" $TESTS
  rc=$?
 fi
 
 # Rootless. (Only if we're not already root)
 if [[ "$TEST_ROOTLESS" && "$(id -u)" != 0 ]]; then
  echo "--------------------------------------------------"
  echo "\$ bats ${bats_opts[*]} ${bats_filter[*]} $TESTS"
- bats "${bats_opts[@]}" "${bats_filter[@]}" $TESTS
+ bats "${bats_opts[@]}" "${bats_filter[@]}" \
+ --gather-test-outputs-in "$BATS_LOGDIR/rootless" $TESTS
  rc=$((rc | $?))
 fi
 

test/NEW-IMAGES

@@ -12,3 +12,4 @@
 #
 # Format is one FQIN per line. Enumerate them below:
 #
+quay.io/libpod/testimage:20241011

test/apiv2/test-apiv2

@@ -12,7 +12,7 @@ ME=$(basename $0)
 PODMAN_TEST_IMAGE_REGISTRY=${PODMAN_TEST_IMAGE_REGISTRY:-"quay.io"}
 PODMAN_TEST_IMAGE_USER=${PODMAN_TEST_IMAGE_USER:-"libpod"}
 PODMAN_TEST_IMAGE_NAME=${PODMAN_TEST_IMAGE_NAME:-"testimage"}
-PODMAN_TEST_IMAGE_TAG=${PODMAN_TEST_IMAGE_TAG:-"20240123"}
+PODMAN_TEST_IMAGE_TAG=${PODMAN_TEST_IMAGE_TAG:-"20241011"}
 PODMAN_TEST_IMAGE_FQN="$PODMAN_TEST_IMAGE_REGISTRY/$PODMAN_TEST_IMAGE_USER/$PODMAN_TEST_IMAGE_NAME:$PODMAN_TEST_IMAGE_TAG"
 
 IMAGE=$PODMAN_TEST_IMAGE_FQN

test/e2e/config_amd64.go

@@ -10,7 +10,7 @@ var (
  NGINX_IMAGE = "quay.io/libpod/alpine_nginx:latest" //nolint:revive,stylecheck
  BB_GLIBC = "docker.io/library/busybox:glibc" //nolint:revive,stylecheck
  REGISTRY_IMAGE = "quay.io/libpod/registry:2.8.2" //nolint:revive,stylecheck
- CITEST_IMAGE = "quay.io/libpod/testimage:20240123" //nolint:revive,stylecheck
+ CITEST_IMAGE = "quay.io/libpod/testimage:20241011" //nolint:revive,stylecheck
  SYSTEMD_IMAGE = "quay.io/libpod/systemd-image:20240124" //nolint:revive,stylecheck
  CIRROS_IMAGE = "quay.io/libpod/cirros:latest" //nolint:revive,stylecheck
 )

test/e2e/config_arm64.go

@@ -10,7 +10,7 @@ var (
  NGINX_IMAGE = "quay.io/lsm5/alpine_nginx-aarch64:latest" //nolint:revive,stylecheck
  BB_GLIBC = "docker.io/library/busybox:glibc" //nolint:revive,stylecheck
  REGISTRY_IMAGE = "quay.io/libpod/registry:2.8.2" //nolint:revive,stylecheck
- CITEST_IMAGE = "quay.io/libpod/testimage:20240123" //nolint:revive,stylecheck
+ CITEST_IMAGE = "quay.io/libpod/testimage:20241011" //nolint:revive,stylecheck
  SYSTEMD_IMAGE = "quay.io/libpod/systemd-image:20240124" //nolint:revive,stylecheck
  CIRROS_IMAGE = "quay.io/libpod/cirros:latest" //nolint:revive,stylecheck
 )

test/e2e/config_ppc64le.go

@@ -9,6 +9,6 @@ var (
  CACHE_IMAGES = []string{ALPINE, BB, fedoraMinimal, NGINX_IMAGE, REDIS_IMAGE, INFRA_IMAGE, CITEST_IMAGE}
  NGINX_IMAGE = "quay.io/libpod/alpine_nginx-ppc64le:latest"
  BB_GLIBC = "docker.io/ppc64le/busybox:glibc"
- CITEST_IMAGE = "quay.io/libpod/testimage:20240123"
+ CITEST_IMAGE = "quay.io/libpod/testimage:20241011"
  REGISTRY_IMAGE string
 )

test/system/030-run.bats

@@ -858,6 +858,7 @@ json-file | f
  # exactly 10 seconds. Give it some leeway.
  delta_t=$(( $t1 - $t0 ))
  assert "$delta_t" -gt 1 "podman stop: ran too quickly!"
+ # FIXME: can fail under load, take 7 seconds
  assert "$delta_t" -le 6 "podman stop: took too long"
 
  run_podman rm $cname

test/system/035-logs.bats

@@ -336,6 +336,14 @@ function _log_test_follow_since() {
  run_podman ${events_backend} run --log-driver=$driver --name $cname -d $IMAGE \
  sh -c "sleep 1; while :; do echo $content && sleep 1; done"
 
+ # FIXME FIXME: TEMPORARY! For debugging a bug I don't even remember any more
+ logpath=
+ if [[ "$driver" = "k8s-file" ]]; then
+ run_podman inspect --format '{{.HostConfig.LogConfig.Path}}' $cname
+ logpath="$output"
+ fi
+ # FIXME FIXME
+
  # sleep is required to make sure the podman event backend no longer sees the start event in the log
  # This value must be greater or equal than the value given in --since below
  sleep 0.2

test/system/080-pause.bats

@@ -48,6 +48,7 @@ load helpers
 
  # There should be a 3-4 second gap, *maybe* 5. Never 1 or 2, that
  # would imply that the container never paused.
+ # FIXME: under high load, can be 7
  is "$max_delta" "[3456]" "delta t between paused and restarted"
 
  run_podman rm -t 0 -f $cname

test/system/220-healthcheck.bats

@@ -82,6 +82,12 @@ Log[-1].ExitCode | 0
 Log[-1].Output | \"Life is Good on stdout\\\nLife is Good on stderr\\\n\"
 " "$current_time" "healthy"
 
+ # FIXME FIXME FIXME: 20240918: there's a race here, wherein _check_health()
+ # can see a "healthy" that comes from before 'touch uh-oh'. One way to
+ # fix that might be to add another arg to _check_health, 'FailingStreak'.
+ # That doesn't show up in podman-events, though, so we'd have to
+ # run podman-inspect in a loop, and that introduces its own races.
+ # I don't have a good answer here. See log.103
  current_time=$(date --iso-8601=ns)
  # Force a failure
  run_podman exec $ctrname touch /uh-oh

test/system/250-systemd.bats

@@ -372,6 +372,7 @@ LISTEN_FDNAMES=listen_fdnames" | sort)
  run_podman exec $cname touch /uh-oh
 
  # healthcheck should now fail, with exit status 1 and 'unhealthy' output
+ # FIXME: race: on high load, we can get "Error: no container with ID xxxx"
  run_podman 1 healthcheck run $cname
  is "$output" "unhealthy" "output from 'podman healthcheck run'"
 

test/system/255-auto-update.bats

@@ -18,14 +18,17 @@ function setup() {
 }
 
 function teardown() {
+ ls -l /run/netns | sed -e "s/^/# teardown /" >&3
  if [[ -e $SNAME_FILE ]]; then
  while read line; do
  if [[ "$line" =~ "podman-auto-update" ]]; then
  echo "Stop timer: $line.timer"
  systemctl stop $line.timer
  systemctl disable $line.timer
  else
+ ls -l /run/netns | sed -e "s/^/# before stop $line /" >&3
  systemctl stop $line
+ ls -l /run/netns | sed -e "s/^/# after stop $line /" >&3
  fi
  rm -f $UNIT_DIR/$line.{service,timer}
  done < $SNAME_FILE
@@ -66,12 +69,12 @@ function generate_service() {
 
  # Unless specified, set a default command.
  if [[ -z "$command" ]]; then
- command="top -d 120"
+ command="top -d $((100 + BATS_SUITE_TEST_NUMBER))"
  fi
 
  # Container name. Include the autoupdate type, to make debugging easier.
  # IMPORTANT: variable 'cname' is passed (out of scope) up to caller!
- cname=c_${autoupdate//\'/}_$(random_string)
+ cname="c-$(safename)-${autoupdate//\'/}-$(random_string)"
  target_img="quay.io/libpod/$target_img_basename:latest"
  if [[ -n "$7" ]]; then
  target_img="$7"
@@ -172,7 +175,7 @@ function _confirm_update() {
 
 # This test can fail in dev. environment because of SELinux.
 # quick fix: chcon -t container_runtime_exec_t ./bin/podman
-@test "podman auto-update - label io.containers.autoupdate=image" {
+@test "podman auto-update - label io.containers.autoupdate=imagexxxxxxx" {
  since=$(date --iso-8601=seconds)
  run_podman auto-update
  is "$output" ""
@@ -214,6 +217,11 @@ function _confirm_update() {
  run_podman container inspect --format "{{.ID}}" $ctr_child
  run_podman container inspect --format "{{.State.Status}}" $ctr_child
  is "$output" "running" "child container is in running state"
+
+ ls -l /run/netns | sed -e 's/^/# before container rm /' >&3
+ run_podman container rm -f -t0 $ctr_child
+ run_podman container rm -f -t0 $ctr_parent
+ ls -l /run/netns | sed -e 's/^/# after container rm /' >&3
 }
 
 @test "podman auto-update - label io.containers.autoupdate=image with rollback" {

test/system/260-sdnotify.bats

@@ -553,8 +553,15 @@ none | false | false | 0
  export NOTIFY_SOCKET=$PODMAN_TMPDIR/notify-$(safename).sock
  _start_socat
 
+ echo "ls -l $PODMAN_TMPDIR/"
+ ls -l $PODMAN_TMPDIR/
+
  run_podman push $registry_flags $IMAGE $image_on_local_registry
 
+ # Again
+ echo "ls -l $PODMAN_TMPDIR/"
+ ls -l $PODMAN_TMPDIR/
+
  run_podman pull $registry_flags $image_on_local_registry
  is "${lines[1]}" "Pulling image //$image_on_local_registry inside systemd: setting pull timeout to 5m0s" "NOTIFY_SOCKET is passed to container"
 

test/system/320-system-df.bats

@@ -45,9 +45,9 @@ function setup_file() {
  local results="$output"
 
  # FIXME! This needs to be fiddled with every time we bump testimage.
- local size=11
+ local size=12
  if [[ "$(uname -m)" = "aarch64" ]]; then
- size=13
+ size=14
  fi
 
  # FIXME: we can't check exact RawSize or Size because every CI system

test/system/331-system-check.bats

@@ -4,9 +4,21 @@
 # that they are caught and remedied, even if it requires discarding some
 # data in read-write layers.
 #
+# DO NOT PARALLELIZE. All of these tests require complete control of images.
+#
 
 load helpers
 
+function setup_file() {
+ # Pristine setup: no pods, containers, volumes, images
+ run_podman pod rm -a -f
+ run_podman rm -f -a -t0
+ run_podman volume rm -a
+ run_podman image rm -f -a
+
+ _prefetch $IMAGE
+}
+
 @test "podman system check - unmanaged layers" {
  run_podman_testing create-storage-layer
  layerID="$output"

test/system/450-interactive.bats

@@ -5,17 +5,19 @@
 
 load helpers
 
+# bats file_tags=ci:parallel
+
 ###############################################################################
 # BEGIN setup/teardown
 
-# Each test runs with its own PTY, managed by socat.
-PODMAN_TEST_PTY=$(mktemp -u --tmpdir=${BATS_TMPDIR:-/tmp} podman_pty.XXXXXX)
-PODMAN_DUMMY=$(mktemp -u --tmpdir=${BATS_TMPDIR:-/tmp} podman_dummy.XXXXXX)
-PODMAN_SOCAT_PID=
-
 function setup() {
  basic_setup
 
+ # Each test runs with its own PTY, managed by socat.
+ PODMAN_TEST_PTY=$PODMAN_TMPDIR/podman_pty
+ PODMAN_DUMMY=$PODMAN_TMPDIR/podman_dummy
+ PODMAN_SOCAT_PID=
+
  # Create a pty. Run under 'timeout' because BATS reaps child processes
  # and if we exit before killing socat, bats will hang forever.
  timeout 10 socat \