diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh
index dc67dc400..4b55db526 100755
--- a/ci/build_wheel.sh
+++ b/ci/build_wheel.sh
@@ -46,10 +46,7 @@ else
     apt install yasm libopenslide-dev -y
     dpkg -L libopenslide-dev
 
-    # temporarily add a sleep here to slow down the aarch64 workers to allow
-    # x86_64 cases to try to run instead of being cancelled if the aarch64
-    # case failed
-    sleep 10m
+    git apply ci/omit-benchmarks-build.patch
 fi
 
 # First build the C++ lib using CMake via the run script
diff --git a/ci/omit-benchmarks-build.patch b/ci/omit-benchmarks-build.patch
new file mode 100644
index 000000000..24b83715b
--- /dev/null
+++ b/ci/omit-benchmarks-build.patch
@@ -0,0 +1,22 @@
+diff --git a/CMakeLists.txt b/CMakeLists.txt
+index 0ab2e17..2a42462 100644
+--- a/CMakeLists.txt
++++ b/CMakeLists.txt
+@@ -138,7 +138,7 @@ set(CUCIM_PACKAGE_NAME cucim) # cucim
+ ################################################################################
+ add_subdirectory(cpp)
+ add_subdirectory(gds)
+-add_subdirectory(benchmarks)
++# add_subdirectory(benchmarks)
+ add_subdirectory(examples/cpp)
+ 
+ ################################################################################
+@@ -155,7 +155,7 @@ configure_file(${CMAKE_CURRENT_LIST_DIR}/examples/cpp/CMakeLists.txt.examples.re
+ set(INSTALL_TARGETS
+         ${CUCIM_PACKAGE_NAME}
+         fmt-header-only
+-        cucim_benchmarks
++        # cucim_benchmarks
+         )
+ 
+ install(TARGETS ${INSTALL_TARGETS}