Run tests on GitHub builds

.github/run_tests.sh

@@ -0,0 +1,240 @@
+#!/bin/bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [[ -z "$GITHUB_STEP_SUMMARY" ]]; then
+  echo "run_tests.sh is for use by CI (selected tests, timeout)."
+  echo "Users should go to opt/cachelib/tests and run make."
+  echo
+fi
+
+# Optional (e.g., flaky) tests. Issues a warning instead of an error.
+OPTIONAL=()
+OPTIONAL+=("allocator-test-AllocationClassTest")  # Ubuntu 18 (segfault)
+OPTIONAL+=("allocator-test-AllocatorResizeTypeTest")  # Rocky 8.6
+OPTIONAL+=("allocator-test-AllocatorTypeTest")  # CentOS 8.5, Rocky 8.6
+OPTIONAL+=("allocator-test-BlockCacheTest")  # Rocky 8.6
+OPTIONAL+=("allocator-test-MemoryAllocatorTest")  # Ubuntu 18 (segfault)
+OPTIONAL+=("allocator-test-MM2QTest")  # Ubuntu 18 (segfault)
+# CentOS 8.1, CentOS 8.5, Debian, Fedora 36, Rocky 9, Rocky 8.6, Ubuntu 18
+OPTIONAL+=("allocator-test-NavySetupTest")
+OPTIONAL+=("allocator-test-NvmCacheTests")  # Rocky 8.6
+OPTIONAL+=("allocator-test-RefCountTest")  # Ubuntu 18
+OPTIONAL+=("allocator-test-SlabAllocatorTest")  # Ubuntu 18
+# CentOS 8.1, CentOS 8.5, Debian, Fedora 36, Rocky 9, Rocky 8.6, Ubuntu 18
+OPTIONAL+=("common-test-UtilTests")
+OPTIONAL+=("datatype-test-MapTest")  # Ubuntu 20
+# CentOS 8.1, Rocky 9, Ubuntu 18
+OPTIONAL+=("navy-test-BlockCacheTest")
+# CentOS 8.1, CentOS 8.5, Debian, Fedora 36, Rocky 9, Ubuntu 18, Ubuntu 20, Ubuntu 22
+OPTIONAL+=("navy-test-DriverTest")  
+# CentOS 8.5, Rocky 9, Ubuntu 20
+OPTIONAL+=("navy-test-MockJobSchedulerTest")
+# CentOS 8.1, CentOS 8.5, Debian, Fedora 36, Rocky 9, Rocky 8.6, Ubuntu 18, Ubuntu 20, Ubuntu 22
+# Large pages need to be enabled
+OPTIONAL+=("shm-test-test_page_size")  
+
+# Skip long-running benchmarks.
+TO_SKIP=()
+# TO_SKIP+=("allocator-test-AllocatorTypeTest")  # 12 mins.
+TO_SKIP+=("benchmark-test-CompactCacheBench")  # 26 mins.
+TO_SKIP+=("benchmark-test-MutexBench")  # 60 mins.
+
+TEST_TIMEOUT=30m
+BENCHMARK_TIMEOUT=20m
+PARALLELISM=10
+
+print_test_log() {
+  logfile=$1
+  # Print last failed test
+  EXP_LOG=$(grep -Pazo \
+      "(?s)\[ RUN[^\[]+\[  FAILED[^\n]+ms\)\n" $logfile \
+    | sed 's/\x0/---------------\n/g')
+  # And contents of last test before core dumps
+  if grep -q -R "core dumped" $logfile; then
+    EXP_LOG+=$'\n'
+    EXP_LOG+=$(tac $logfile | sed '/\[ RUN      \]/q' | tac)
+  fi
+  echo "::group::Logs:$logfile"
+  echo "$EXP_LOG"
+  echo
+  echo "::endgroup::"
+  echo
+
+  echo "#### $logfile" >> $MD_OUT
+  echo "\`\`\`" >> $MD_OUT
+  echo "$EXP_LOG" >> $MD_OUT
+  echo "\`\`\`" >> $MD_OUT
+  echo >> $MD_OUT
+}
+
+OPTIONAL_LIST=$(printf  -- '%s\n' ${OPTIONAL[@]})
+TO_SKIP_LIST=$(printf  -- '%s\n' ${TO_SKIP[@]})
+
+MD_OUT=${GITHUB_STEP_SUMMARY:-$PWD/summary.md}
+if [[ "$MD_OUT" != "$GITHUB_STEP_SUMMARY" ]]; then
+  echo "Markdown summary will be saved in $MD_OUT. Truncating it."
+  echo
+  echo "Time started: $(date)" > $MD_OUT
+fi
+
+echo "See Summary page of job for a table of test results and log excerpts."
+echo 
+
+dir=$(dirname "$0")
+cd "$dir/.." || die "failed to change-dir into $dir/.."
+test -d cachelib || die "failed to change-dir to expected root directory"
+
+PREFIX="$PWD/opt/cachelib"
+LD_LIBRARY_PATH="$PREFIX/lib:${LD_LIBRARY_PATH:-}"
+export LD_LIBRARY_PATH
+
+echo "LD_LIBRARY_PATH: $LD_LIBRARY_PATH"
+
+cd opt/cachelib/tests || die "failed to change-dir into opt/cachelib/tests"
+
+TESTS_TO_RUN=$(find * -type f -not -name "*bench*" -executable \
+  | grep -vF "$TO_SKIP_LIST" \
+  | awk ' { print $1 ".log" } ')
+N_TESTS=$(echo $TESTS_TO_RUN | wc -w)
+
+if [[ $(< /proc/sys/vm/nr_hugepages) == "0" ]]; then
+  # GitHub's runners have 7GB of RAM (as of 2023)
+  echo
+  echo "Trying to allocate a 1GB huge page pool for shm-test-test_page_size"
+  sudo sysctl -w vm.nr_hugepages=512
+fi
+
+echo
+echo "::group::Running tests for CI (total: $N_TESTS, max: $TEST_TIMEOUT)"
+timeout --preserve-status $TEST_TIMEOUT make -j $PARALLELISM -s $TESTS_TO_RUN
+echo "::endgroup::"
+echo "Successful tests: $(find -name '*.ok' | wc -l)"
+echo "Failed tests: $(find -name '*.fail' | wc -l)"
+echo
+
+BENCHMARKS_TO_RUN=$(find * -type f -name "*bench*" -executable \
+  | grep -vF "$TO_SKIP_LIST" \
+  | awk ' { print $1 ".log" } ')
+N_BENCHMARKS=$(echo $BENCHMARKS_TO_RUN | wc -w)
+
+echo "::group::Running benchmarks for CI (total: $N_BENCHMARKS, max: $BENCHMARK_TIMEOUT)"
+timeout --preserve-status $BENCHMARK_TIMEOUT make -j $PARALLELISM -s $BENCHMARKS_TO_RUN
+echo "::endgroup::"
+echo "Successful benchmarks: $(find -name '*bench*.ok' | wc -l)"
+echo "Failed benchmarks: $(find -name '*bench*.fail' | wc -l)"
+
+TESTS_PASSED=$(find * -name '*.log.ok' | sed 's/\.log\.ok$//')
+TESTS_FAILED=$(find * -name '*.log.fail' | sed 's/\.log\.fail$//')
+TESTS_TIMEOUT=$(find * -type f -executable \
+  | grep -vF "$TESTS_PASSED" \
+  | grep -vF "$TESTS_FAILED" \
+  | grep -vF "$TO_SKIP_LIST")
+TESTS_IGNORED=$(echo "$TESTS_FAILED" | grep -F "$OPTIONAL_LIST")
+FAILURES_UNIGNORED=$(echo "$TESTS_FAILED" | grep -vF "$OPTIONAL_LIST")
+
+N_TIMEOUT=$(echo $TESTS_TIMEOUT | wc -w)
+N_PASSED=$(echo $TESTS_PASSED | wc -w)
+N_FAILED=$(echo $TESTS_FAILED | wc -w)
+N_IGNORED=$(echo $TESTS_IGNORED | wc -w)
+N_FAILURES_UNIGNORED=$(echo $FAILURES_UNIGNORED | wc -w)
+N_SKIPPED=$(echo $TO_SKIP_LIST | wc -w)
+
+echo "## Test summary" >> $MD_OUT
+echo "|Workflow| Passed | Failed | Ignored | Timeout | Skipped" >> $MD_OUT
+echo "|--------|--------|--------|---------|---------|---------|" >> $MD_OUT
+echo "| $GITHUB_JOB | $N_PASSED | $N_FAILED | $N_IGNORED | $N_TIMEOUT | $N_SKIPPED |" >> $MD_OUT
+
+STATUS=0
+
+if [[ $N_FAILED -ne 0 ]]; then
+
+  echo
+  echo "::group::Failures at a glance"
+  grep "core dumped" *.log || true
+  grep "FAILED.*ms" *.log || true
+  echo "::endgroup::"
+
+  echo >> $MD_OUT
+  echo "## Failures at a glance" >> $MD_OUT
+  echo "\`\`\`" >> $MD_OUT
+  grep "core dumped" *.log >> $MD_OUT || true
+  grep "FAILED.*ms" *.log >> $MD_OUT || true
+  echo "\`\`\`" >> $MD_OUT
+
+  if [ $N_FAILURES_UNIGNORED -eq 0 ]; then
+    echo "Only ignored tests failed."
+  else
+    STATUS=1
+    echo
+    echo "== Failing tests =="
+    echo "::error ::$N_FAILURES_UNIGNORED tests/benchmarks failed."
+
+    echo "$FAILURES_UNIGNORED"
+
+    echo >> $MD_OUT
+    echo "## Failing tests" >> $MD_OUT
+    echo "$FAILURES_UNIGNORED" | awk ' { print "1. " $1 } ' >> $MD_OUT
+
+
+    for failedtest in $FAILURES_UNIGNORED; do
+      echo "::error ::$failedtest failed. See job summary or log for details."
+      print_test_log "$failedtest.log"
+    done
+  fi
+
+  if [[ $N_IGNORED -ne 0 ]]; then
+    echo
+    echo "::group::Ignored test failures "
+    echo "$TESTS_IGNORED"
+    echo "::endgroup::"
+    echo "::warning ::$N_IGNORED tests/benchmarks failed and ignored."
+
+    echo >> $MD_OUT
+    echo "## Ignored test failures" >> $MD_OUT
+    echo "$TESTS_IGNORED" | awk ' { print "1. " $1 } ' >> $MD_OUT
+
+    for failedtest in $TESTS_IGNORED; do
+      echo "::warning ::$failedtest failed & ignored. See job summary or log for details."
+      print_test_log "$failedtest.log"
+    done
+  fi 
+else
+  echo
+  echo "All tests passed."
+fi
+
+echo
+echo "::group::Skipped tests"
+echo "$TO_SKIP_LIST"
+echo "::endgroup::"
+
+echo >> $MD_OUT
+echo "## Skipped tests" >> $MD_OUT
+echo "$TO_SKIP_LIST" | awk ' { print "1. " $1 } ' >> $MD_OUT
+
+if [[ $N_TIMEOUT -ne 0 ]]; then
+  echo
+  echo "::error ::$N_TIMEOUT tests exceeded time limit." \
+    " Consider adding them to TO_SKIP or increasing TEST_TIMEOUT/BENCHMARK_TIMEOUT."
+  echo "::group::Timed out tests"
+  echo "$TESTS_TIMEOUT"
+  echo "::endgroup::"
+
+  echo "## Tests timed out" >> $MD_OUT
+  echo "$TESTS_TIMEOUT" | awk ' { print "1. " $1 } ' >> $MD_OUT
+fi
+
+# Comment out if you do not want (unignored) failing tests to fail the build
+exit $STATUS

.github/workflows/build-cachelib-centos-8-1.yml

@@ -20,7 +20,7 @@ on:
 jobs:
   build-cachelib-centos8-1-1911:
     #if: "!contains(github.event.head_commit.author.name, 'svcscm')"
-    name: "CentOS/8.1.1911 - Build CacheLib with all dependencies"
+    name: "CentOS/8.1.1911 Build & Test"
     runs-on: ubuntu-latest
     # Docker container image name
     container: "centos:8.1.1911"
@@ -73,11 +73,21 @@ jobs:
           echo === gcc -v ===
           gcc -v
       - name: "checkout sources"
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       - name: "Install Prerequisites"
         run: ./contrib/build.sh -S -B
       - name: "Test: update-submodules"
         run: ./contrib/update-submodules.sh
+      - name: "Use cache for built libraries of submodules"
+        uses: actions/cache@v3
+        id: cache-submodules
+        with:
+          key: ${{ github.job }}-${{ hashFiles('.git/modules/*/HEAD') }}
+          path: |
+            opt/cachelib/lib
+            opt/cachelib/include
+            opt/cachelib/bin/thrift1
+            opt/cachelib/bin/fizz*
       - name: "Install dependency: zstd"
         run: ./contrib/build-package.sh -j -v -i zstd
       - name: "Install dependency: googleflags"
@@ -92,16 +102,22 @@ jobs:
         run: ./contrib/build-package.sh -j -v -i fmt
       - name: "Install dependency: folly"
         run: ./contrib/build-package.sh -j -v -i folly
+        if: steps.cache-submodules.outputs.cache-hit != 'true'
       - name: "Install dependency: fizz"
         run: ./contrib/build-package.sh -j -v -i fizz
+        if: steps.cache-submodules.outputs.cache-hit != 'true'
       - name: "Install dependency: wangle"
         run: ./contrib/build-package.sh -j -v -i wangle
+        if: steps.cache-submodules.outputs.cache-hit != 'true'
       - name: "Install dependency: fbthrift"
         run: ./contrib/build-package.sh -j -v -i fbthrift
+        if: steps.cache-submodules.outputs.cache-hit != 'true'
       - name: "build CacheLib"
         # Build cachelib in debug mode (-d) and with all tests (-t)
         run: ./contrib/build-package.sh -j -v -i -d -t cachelib
-      - uses: actions/upload-artifact@v2
+      - name: "Run tests"
+        run: ./.github/run_tests.sh
+      - uses: actions/upload-artifact@v3
         if: failure()
         with:
           name: cachelib-cmake-logs
@@ -112,3 +128,10 @@ jobs:
             build-cachelib/**/Makefile
           if-no-files-found: warn
           retention-days: 1
+      - uses: actions/upload-artifact@v3
+        with:
+          name: cachelib-test-logs
+          path: opt/cachelib/tests/*.log
+          if-no-files-found: warn
+          retention-days: 7
+        if: success() || failure()

.github/workflows/build-cachelib-centos-8-5.yml

@@ -19,7 +19,7 @@ on:
      - cron:  '0 9 * * *'
 jobs:
   build-cachelib-centos8-latest:
-    name: "CentOS/8.5 - Build CacheLib with all dependencies"
+    name: "CentOS/8.5 Build & Test"
     runs-on: ubuntu-latest
     # Docker container image name
     container: "centos:latest"
@@ -72,11 +72,21 @@ jobs:
           echo === gcc -v ===
           gcc -v
       - name: "checkout sources"
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
       - name: "Install Prerequisites"
         run: ./contrib/build.sh -S -B
       - name: "Test: update-submodules"
         run: ./contrib/update-submodules.sh
+      - name: "Use cache for built libraries of submodules"
+        uses: actions/cache@v3
+        id: cache-submodules
+        with:
+          key: ${{ github.job }}-${{ hashFiles('.git/modules/*/HEAD') }}
+          path: |
+            opt/cachelib/lib
+            opt/cachelib/include
+            opt/cachelib/bin/thrift1
+            opt/cachelib/bin/fizz*
       - name: "Install dependency: zstd"
         run: ./contrib/build-package.sh -j -v -i zstd
       - name: "Install dependency: googleflags"
@@ -91,16 +101,22 @@ jobs:
         run: ./contrib/build-package.sh -j -v -i fmt
       - name: "Install dependency: folly"
         run: ./contrib/build-package.sh -j -v -i folly
+        if: steps.cache-submodules.outputs.cache-hit != 'true'
       - name: "Install dependency: fizz"
         run: ./contrib/build-package.sh -j -v -i fizz
+        if: steps.cache-submodules.outputs.cache-hit != 'true'
       - name: "Install dependency: wangle"
         run: ./contrib/build-package.sh -j -v -i wangle
+        if: steps.cache-submodules.outputs.cache-hit != 'true'
       - name: "Install dependency: fbthrift"
         run: ./contrib/build-package.sh -j -v -i fbthrift
+        if: steps.cache-submodules.outputs.cache-hit != 'true'
       - name: "build CacheLib"
         # Build cachelib in debug mode (-d) and with all tests (-t)
         run: ./contrib/build-package.sh -j -v -i -d -t cachelib
-      - uses: actions/upload-artifact@v2
+      - name: "Run tests"
+        run: ./.github/run_tests.sh
+      - uses: actions/upload-artifact@v3
         if: failure()
         with:
           name: cachelib-cmake-logs
@@ -111,3 +127,10 @@ jobs:
             build-cachelib/**/Makefile
           if-no-files-found: warn
           retention-days: 1
+      - uses: actions/upload-artifact@v3
+        with:
+          name: cachelib-test-logs
+          path: opt/cachelib/tests/*.log
+          if-no-files-found: warn
+          retention-days: 7
+        if: success() || failure()