WIP: system test parallelization: two-pass approach

Split system tests into two: those that can be run in
parallel, and those that can't. Run tests in two passes.
This requires eliminating the per-test leak check and
teardown. I think that's okay.

Tests that can run in parallel:

  * use unique container/pod/volume/network names
    - bonus: added a way to track names to their test,
      so the leak test at end can be useful
  * do not run 'podman rm -a' or 'rmi -a'
  * do not run 'podman ps/images' and expect precise output

Signed-off-by: Ed Santiago <[email protected]>

Makefile

@@ -681,7 +681,8 @@ localmachine:
 localsystem:
  # Wipe existing config, database, and cache: start with clean slate.
  $(RM) -rf ${HOME}/.local/share/containers ${HOME}/.config/containers
- if timeout -v 1 true; then PODMAN=$(CURDIR)/bin/podman QUADLET=$(CURDIR)/bin/quadlet bats -T test/system/; else echo "Skipping $@: 'timeout -v' unavailable'"; fi
+ PODMAN=$(CURDIR)/bin/podman QUADLET=$(CURDIR)/bin/quadlet bats -T --filter-tags '!para' test/system/
+ PODMAN=$(CURDIR)/bin/podman QUADLET=$(CURDIR)/bin/quadlet bats -T --filter-tags para -j $$(nproc) test/system/
 
 .PHONY: remotesystem
 remotesystem:
@@ -706,7 +707,8 @@ remotesystem:
  echo "Error: ./bin/podman system service did not come up on $$SOCK_FILE" >&2;\
  exit 1;\
  fi;\
- env PODMAN="$(CURDIR)/bin/podman-remote --url $$PODMAN_SOCKET" bats -T test/system/ ;\
+ env PODMAN="$(CURDIR)/bin/podman-remote --url $$PODMAN_SOCKET" bats -T --filter-tags '!para' test/system/ ;\
+ env PODMAN="$(CURDIR)/bin/podman-remote --url $$PODMAN_SOCKET" bats -T --filter-tags para -j $$(nproc) test/system/ ;\
  rc=$$?;\
  kill %1;\
  rm -f $$SOCK_FILE;\

hack/bats

@@ -83,7 +83,10 @@ for i;do
  --rootless) TEST_ROOT= ;;
  --remote) REMOTE=remote ;;
  --ts|-T) bats_opts+=("-T") ;;
- --tag=*) bats_filter=("--filter-tags" "$value") ;;
+ --tag=*) bats_filter=("--filter-tags" "$value")
+ if [[ "$value" = "para" ]]; then
+ bats_opts+=("--jobs" $(nproc))
+ fi;;
  */*.bats) TESTS=$i ;;
  *)
  if [[ $i =~ : ]]; then

test/system/030-run.bats

@@ -3,7 +3,7 @@
 load helpers
 load helpers.network
 
-# bats test_tags=distro-integration
+# bats test_tags=distro-integration,para
 @test "podman run - basic tests" {
  rand=$(random_string 30)
 
@@ -50,12 +50,14 @@ echo $rand | 0 | $rand
  is "$tests_run" "$(grep . <<<$tests | wc -l)" "Ran the full set of tests"
 }
 
+# bats test_tags=para
 @test "podman run - global runtime option" {
  skip_if_remote "runtime flag is not passed over remote"
  run_podman 126 --runtime-flag invalidflag run --rm $IMAGE
  is "$output" ".*invalidflag" "failed when passing undefined flags to the runtime"
 }
 
+# bats test_tags=para
 @test "podman run --memory=0 runtime option" {
  run_podman run --memory=0 --rm $IMAGE echo hello
  if is_rootless && ! is_cgroupsv2; then
@@ -67,6 +69,7 @@ echo $rand | 0 | $rand
 }
 
 # 'run --preserve-fds' passes a number of additional file descriptors into the container
+# bats test_tags=para
 @test "podman run --preserve-fds" {
  skip_if_remote "preserve-fds is meaningless over remote"
 
@@ -78,6 +81,7 @@ echo $rand | 0 | $rand
 }
 
 # 'run --preserve-fd' passes a list of additional file descriptors into the container
+# bats test_tags=para
 @test "podman run --preserve-fd" {
  skip_if_remote "preserve-fd is meaningless over remote"
 
@@ -96,6 +100,7 @@ echo $rand | 0 | $rand
  assert "${lines[2]}" = "$content" "cat from fd 40"
 }
 
+# bats test_tags=para
 @test "podman run - uidmapping has no /sys/kernel mounts" {
  skip_if_cgroupsv1 "run --uidmap fails on cgroups v1 (issue 15025, wontfix)"
  skip_if_rootless "cannot umount as rootless"
@@ -111,6 +116,7 @@ echo $rand | 0 | $rand
 
 # 'run --rm' goes through different code paths and may lose exit status.
 # See https://github.com/containers/podman/issues/3795
+# bats test_tags=para
 @test "podman run --rm" {
 
  run_podman 0 run --rm $IMAGE /bin/true
@@ -121,14 +127,15 @@ echo $rand | 0 | $rand
  run_podman 1 run --rm $IMAGE sh -c /bin/false
 }
 
+# bats test_tags=para
 @test "podman run --name" {
- randomname=$(random_string 30)
+ randomname=c$(safename)
 
  run_podman run -d --name $randomname $IMAGE sleep inf
  cid=$output
 
  run_podman ps --format '{{.Names}}--{{.ID}}'
- is "$output" "$randomname--${cid:0:12}"
+ assert "$output" =~ "$randomname--${cid:0:12}"
 
  run_podman container exists $randomname
  run_podman container exists $cid
@@ -146,6 +153,7 @@ echo $rand | 0 | $rand
  run_podman 1 container exists $cid
 }
 
+# not parallelizable due to podman rm -a at end
 @test "podman run --pull" {
  run_podman run --pull=missing $IMAGE true
  is "$output" "" "--pull=missing [present]: no output"
@@ -186,6 +194,7 @@ echo $rand | 0 | $rand
 }
 
 # 'run --rmi' deletes the image in the end unless it's used by another container
+# bats test_tags=para
 @test "podman run --rmi" {
  # Name of a nonlocal image. It should be pulled in by the first 'run'
  NONLOCAL_IMAGE="$PODMAN_NONLOCAL_IMAGE_FQN"
@@ -212,6 +221,7 @@ echo $rand | 0 | $rand
 
 # 'run --conmon-pidfile --cid-file' makes sure we don't regress on these flags.
 # Both are critical for systemd units.
+# bats test_tags=para
 @test "podman run --conmon-pidfile --cidfile" {
  pidfile=${PODMAN_TMPDIR}/pidfile
  cidfile=${PODMAN_TMPDIR}/cidfile
@@ -220,7 +230,7 @@ echo $rand | 0 | $rand
  # on write.
  echo "$(random_string 120)" > $cidfile
 
- cname=$(random_string)
+ cname=c$(safename)
  run_podman run --name $cname \
  --conmon-pidfile=$pidfile \
  --cidfile=$cidfile \
@@ -251,36 +261,39 @@ echo $rand | 0 | $rand
  fi
 }
 
+# bats test_tags=para
 @test "podman run docker-archive" {
  skip_if_remote "podman-remote does not support docker-archive"
 
  # Create an image that, when run, outputs a random magic string
+ cname=c$(safename)
  expect=$(random_string 20)
- run_podman run --name myc --entrypoint="[\"/bin/echo\",\"$expect\"]" $IMAGE
+ run_podman run --name $cname --entrypoint="[\"/bin/echo\",\"$expect\"]" $IMAGE
  is "$output" "$expect" "podman run --entrypoint echo-randomstring"
 
  # Save it as a tar archive
- run_podman commit myc myi
+ iname=i$(safename)
+ run_podman commit $cname $iname
  archive=$PODMAN_TMPDIR/archive.tar
- run_podman save --quiet myi -o $archive
+ run_podman save --quiet $iname -o $archive
  is "$output" "" "podman save"
 
  # Clean up image and container from container storage...
- run_podman rmi myi
- run_podman rm myc
+ run_podman rmi $iname
+ run_podman rm $cname
 
  # ... then confirm we can run from archive. This re-imports the image
  # and runs it, producing our random string as the last line.
- run_podman run docker-archive:$archive
+ run_podman run --name $cname docker-archive:$archive
  is "${lines[0]}" "Getting image source signatures" "podman run docker-archive, first line of output"
  is "$output" ".*Copying blob" "podman run docker-archive"
  is "$output" ".*Copying config" "podman run docker-archive"
  is "$output" ".*Writing manifest" "podman run docker-archive"
  is "${lines[-1]}" "$expect" "podman run docker-archive: expected random string output"
 
  # Clean up container as well as re-imported image
- run_podman rm -a
- run_podman rmi myi
+ run_podman rm $cname
+ run_podman rmi $iname
 
  # Repeat the above, with podman-create and podman-start.
  run_podman create docker-archive:$archive
@@ -291,13 +304,14 @@ echo $rand | 0 | $rand
 
  # Clean up.
  run_podman rm $cid
- run_podman rmi myi
+ run_podman rmi $iname
 }
 
 # #6735 : complex interactions with multiple user namespaces
 # The initial report has to do with bind mounts, but that particular
 # symptom only manifests on a fedora container image -- we have no
 # reproducer on alpine. Checking directory ownership is good enough.
+# bats test_tags=para
 @test "podman run : user namespace preserved root ownership" {
  keep="--userns=keep-id"
  is_rootless || keep=""
@@ -315,7 +329,7 @@ echo $rand | 0 | $rand
 }
 
 # #6829 : add username to /etc/passwd inside container if --userns=keep-id
-# bats test_tags=distro-integration
+# bats test_tags=distro-integration, para
 @test "podman run : add username to /etc/passwd if --userns=keep-id" {
  skip_if_not_rootless "--userns=keep-id only works in rootless mode"
  # Default: always run as root
@@ -367,6 +381,7 @@ echo $rand | 0 | $rand
 }
 
 # #6991 : /etc/passwd is modifiable
+# bats test_tags=para
 @test "podman run : --userns=keep-id: passwd file is modifiable" {
  skip_if_not_rootless "--userns=keep-id only works in rootless mode"
  run_podman run -d --userns=keep-id --cap-add=dac_override $IMAGE top
@@ -394,6 +409,7 @@ echo $rand | 0 | $rand
 }
 
 # For #7754: json-file was equating to 'none'
+# bats test_tags=para
 @test "podman run --log-driver" {
  # '-' means that LogPath will be blank and there's no easy way to test
  tests="
@@ -407,15 +423,16 @@ json-file | f
 
  while read driver do_check; do
  msg=$(random_string 15)
- run_podman run --name myctr --log-driver $driver $IMAGE echo $msg
+ cname=c$(safename)
+ run_podman run --name $cname --log-driver $driver $IMAGE echo $msg
  is "$output" "$msg" "basic output sanity check (driver=$driver)"
 
  # Simply confirm that podman preserved our argument as-is
- run_podman inspect --format '{{.HostConfig.LogConfig.Type}}' myctr
+ run_podman inspect --format '{{.HostConfig.LogConfig.Type}}' $cname
  is "$output" "$driver" "podman inspect: driver"
 
  # If LogPath is non-null, check that it exists and has a valid log
- run_podman inspect --format '{{.HostConfig.LogConfig.Path}}' myctr
+ run_podman inspect --format '{{.HostConfig.LogConfig.Path}}' $cname
  if [[ $do_check != '-' ]]; then
  is "$output" "/.*" "LogPath (driver=$driver)"
  if ! test -e "$output"; then
@@ -433,17 +450,17 @@ json-file | f
  # Cannot perform check
  :
  else
- run_podman logs myctr
+ run_podman logs $cname
  is "$output" "$msg" "podman logs, with driver '$driver'"
  fi
  else
- run_podman 125 logs myctr
+ run_podman 125 logs $cname
  if ! is_remote; then
  is "$output" ".*this container is using the 'none' log driver, cannot read logs.*" \
  "podman logs, with driver 'none', should fail with error"
  fi
  fi
- run_podman rm myctr
+ run_podman rm $cname
  done < <(parse_table "$tests")
 
  # Invalid log-driver argument
@@ -452,6 +469,7 @@ json-file | f
  "--log-driver InvalidDriver"
 }
 
+# bats test_tags=para
 @test "podman run --log-driver journald" {
  skip_if_remote "We cannot read journalctl over remote."
 
@@ -462,15 +480,17 @@ json-file | f
  pidfile="${PODMAN_TMPDIR}/$(random_string 20)"
 
  # Multiple --log-driver options to confirm that last one wins
- run_podman run --name myctr --log-driver=none --log-driver journald \
+ cname=c_$(safename)
+ run_podman run --name $cname --log-driver=none --log-driver journald \
  --conmon-pidfile $pidfile $IMAGE echo $msg
 
  journalctl --output cat _PID=$(cat $pidfile)
  is "$output" "$msg" "check that journalctl output equals the container output"
 
- run_podman rm myctr
+ run_podman rm $cname
 }
 
+# bats test_tags=para
 @test "podman run --tz" {
  # This file will always have a constant reference timestamp
  local testfile=/home/podman/testimage-id
@@ -497,6 +517,7 @@ json-file | f
  is "$output" "$expect" "podman run with --tz=local, matches host"
 }
 
+# bats test_tags=para
 @test "podman run --tz with zoneinfo" {
  _prefetch $SYSTEMD_IMAGE
 
@@ -508,6 +529,7 @@ json-file | f
 }
 
 # run with --runtime should preserve the named runtime
+# bats test_tags=para
 @test "podman run : full path to --runtime is preserved" {
  skip_if_remote "podman-remote does not support --runtime option"
 
@@ -530,34 +552,40 @@ json-file | f
  rm -f $new_runtime
 }
 
+# bats test_tags=para
 @test "podman --noout run should print output" {
- run_podman --noout run -d --name test $IMAGE echo hi
+ cname=c_$(safename)
+ run_podman --noout run -d --name $cname $IMAGE echo hi
  is "$output" "" "output should be empty"
- run_podman wait test
- run_podman --noout rm test
+ run_podman wait $cname
+ run_podman --noout rm $cname
  is "$output" "" "output should be empty"
 }
 
+# bats test_tags=para
 @test "podman --noout create should print output" {
- run_podman --noout create --name test $IMAGE echo hi
+ cname=c_$(safename)
+ run_podman --noout create --name $cname $IMAGE echo hi
  is "$output" "" "output should be empty"
- run_podman --noout rm test
+ run_podman --noout rm $cname
  is "$output" "" "output should be empty"
 }
 
+# bats test_tags=para
 @test "podman --out run should save the container id" {
  outfile=${PODMAN_TMPDIR}/out-results
 
  # first we'll need to run something, write its output to a file, and then read its contents.
- run_podman --out $outfile run -d --name test $IMAGE echo hola
+ cname=c_$(safename)
+ run_podman --out $outfile run -d --name $cname $IMAGE echo hola
  is "$output" "" "output should be redirected"
- run_podman wait test
+ run_podman wait $cname
 
  # compare the container id against the one in the file
- run_podman container inspect --format '{{.Id}}' test
+ run_podman container inspect --format '{{.Id}}' $cname
  is "$output" "$(<$outfile)" "container id should match"
 
- run_podman --out /dev/null rm test
+ run_podman --out /dev/null rm $cname
  is "$output" "" "output should be empty"
 }
 
@@ -1432,20 +1460,21 @@ search | $IMAGE |
  assert "$output" = "$randcontent" "mounts should appear in container"
 }
 
+# bats file_tags=para
 @test "podman run - rm pod if container creation failed with -pod new:" {
- run_podman run -d --name foobar $IMAGE hostname
+ cname=c_$(safename)
+ run_podman run -d --name $cname $IMAGE hostname
  cid=$output
 
- podname=pod$(random_string)
- run_podman 125 run --rm --pod "new:$podname" --name foobar $IMAGE hostname
- is "$output" ".*creating container storage: the container name \"foobar\" is already in use by"
+ podname=p_$(safename)
+ run_podman 125 run --rm --pod "new:$podname" --name $cname $IMAGE hostname
+ is "$output" ".*creating container storage: the container name \"$cname\" is already in use by"
 
  # pod should've been cleaned up
  # if container creation failed
  run_podman 1 pod exists $podname
 
  run_podman rm $cid
- run_podman rmi $(pause_image)
 }
 
 @test "podman run - no entrypoint" {