From 8b2757dbdbf18b21916d1c08725fba91d69f81f2 Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Tue, 29 Oct 2024 11:22:20 +0900 Subject: [PATCH] Improve retry for `scalardb` test (#131) * Add upper limit to `exponential-backoff` * Increase max retry * Fix unit test * Use the constants in core_test.cli --- scalardb/src/scalardb/core.clj | 9 +++++++-- scalardb/test/scalardb/core_test.clj | 13 ++++++++++--- scalardb/test/scalardb/transfer_2pc_test.clj | 6 ++++-- scalardb/test/scalardb/transfer_append_2pc_test.clj | 3 ++- scalardb/test/scalardb/transfer_append_test.clj | 3 ++- scalardb/test/scalardb/transfer_test.clj | 6 ++++-- 6 files changed, 29 insertions(+), 11 deletions(-) diff --git a/scalardb/src/scalardb/core.clj b/scalardb/src/scalardb/core.clj index 2ebd860..9f77f0b 100644 --- a/scalardb/src/scalardb/core.clj +++ b/scalardb/src/scalardb/core.clj @@ -12,18 +12,23 @@ StorageFactory) (com.scalar.db.transaction.consensuscommit Coordinator))) -(def ^:const RETRIES 8) +(def ^:const RETRIES 20) (def ^:const RETRIES_FOR_RECONNECTION 3) (def ^:private ^:const NUM_FAILURES_FOR_RECONNECTION 1000) +(def ^:private ^:const MAX_WAIT_MILLIS 32000) (def ^:const INITIAL_TABLE_ID 1) (def ^:const DEFAULT_TABLE_COUNT 3) (def ^:const KEYSPACE "jepsen") (def ^:const VERSION "tx_version") +(defn compute-exponential-backoff + [r] + (min MAX_WAIT_MILLIS (reduce * 1000 (repeat r 2)))) + (defn exponential-backoff [r] - (Thread/sleep (reduce * 1000 (repeat r 2)))) + (Thread/sleep (compute-exponential-backoff r))) (defn- get-cassandra-schema "Only the current test schemata are covered diff --git a/scalardb/test/scalardb/core_test.clj b/scalardb/test/scalardb/core_test.clj index 00d00de..1ee27b2 100644 --- a/scalardb/test/scalardb/core_test.clj +++ b/scalardb/test/scalardb/core_test.clj @@ -103,7 +103,7 @@ scalar/create-service-instance (spy/stub nil)] (let [test {:db mock-db :storage (atom nil)}] (scalar/prepare-storage-service! test) - (is (spy/called-n-times? scalar/exponential-backoff 8)) + (is (spy/called-n-times? scalar/exponential-backoff scalardb.core/RETRIES)) (is (nil? @(:storage test)))))) (deftest check-connection-test @@ -150,5 +150,12 @@ (let [test {:storage (atom mock-storage)}] (is (thrown? clojure.lang.ExceptionInfo (scalar/check-transaction-states test #{"tx"}))) - (is (spy/called-n-times? scalar/exponential-backoff 8)) - (is (spy/called-n-times? scalar/prepare-storage-service! 3))))) + (is (spy/called-n-times? scalar/exponential-backoff scalardb.core/RETRIES)) + (is (spy/called-n-times? scalar/prepare-storage-service! + (+ (quot scalar/RETRIES scalar/RETRIES_FOR_RECONNECTION) 1)))))) + +(deftest compute-exponential-backoff-test + (is (= 2000 (scalar/compute-exponential-backoff 1))) + (is (= 4000 (scalar/compute-exponential-backoff 2))) + (is (= 32000 (scalar/compute-exponential-backoff 5))) + (is (= 32000 (scalar/compute-exponential-backoff 10)))) diff --git a/scalardb/test/scalardb/transfer_2pc_test.clj b/scalardb/test/scalardb/transfer_2pc_test.clj index 09a82f6..6ad953b 100644 --- a/scalardb/test/scalardb/transfer_2pc_test.clj +++ b/scalardb/test/scalardb/transfer_2pc_test.clj @@ -235,8 +235,10 @@ (#'transfer/get-all {:client client} nil)))) (is (spy/called-n-times? scalar/exponential-backoff scalar/RETRIES)) - (is (spy/called-n-times? scalar/prepare-transaction-service! scalar/RETRIES_FOR_RECONNECTION)) - (is (spy/called-n-times? scalar/prepare-storage-service! scalar/RETRIES_FOR_RECONNECTION))))) + (is (spy/called-n-times? scalar/prepare-transaction-service! + (+ (quot scalar/RETRIES scalar/RETRIES_FOR_RECONNECTION) 1))) + (is (spy/called-n-times? scalar/prepare-storage-service! + (+ (quot scalar/RETRIES scalar/RETRIES_FOR_RECONNECTION) 1)))))) (deftest transfer-client-check-tx-test (with-redefs [scalar/check-transaction-states (spy/stub 1)] diff --git a/scalardb/test/scalardb/transfer_append_2pc_test.clj b/scalardb/test/scalardb/transfer_append_2pc_test.clj index b4db93f..c7f0844 100644 --- a/scalardb/test/scalardb/transfer_append_2pc_test.clj +++ b/scalardb/test/scalardb/transfer_append_2pc_test.clj @@ -242,7 +242,8 @@ (client/invoke! client {:db mock-db} (transfer/get-all {:client client} nil)))) (is (spy/called-n-times? scalar/exponential-backoff scalar/RETRIES)) - (is (spy/called-n-times? scalar/prepare-transaction-service! scalar/RETRIES_FOR_RECONNECTION))))) + (is (spy/called-n-times? scalar/prepare-transaction-service! + (+ (quot scalar/RETRIES scalar/RETRIES_FOR_RECONNECTION) 1)))))) (deftest transfer-client-check-tx-test (with-redefs [scalar/check-transaction-states (spy/stub 1)] diff --git a/scalardb/test/scalardb/transfer_append_test.clj b/scalardb/test/scalardb/transfer_append_test.clj index 8ef72d0..a265b2c 100644 --- a/scalardb/test/scalardb/transfer_append_test.clj +++ b/scalardb/test/scalardb/transfer_append_test.clj @@ -212,7 +212,8 @@ (client/invoke! client {:db mock-db} (transfer/get-all {:client client} nil)))) (is (spy/called-n-times? scalar/exponential-backoff scalar/RETRIES)) - (is (spy/called-n-times? scalar/prepare-transaction-service! scalar/RETRIES_FOR_RECONNECTION))))) + (is (spy/called-n-times? scalar/prepare-transaction-service! + (+ (quot scalar/RETRIES scalar/RETRIES_FOR_RECONNECTION) 1)))))) (deftest transfer-client-check-tx-test (with-redefs [scalar/check-transaction-states (spy/stub 1)] diff --git a/scalardb/test/scalardb/transfer_test.clj b/scalardb/test/scalardb/transfer_test.clj index f4a9cb2..b750c60 100644 --- a/scalardb/test/scalardb/transfer_test.clj +++ b/scalardb/test/scalardb/transfer_test.clj @@ -205,8 +205,10 @@ (#'transfer/get-all {:client client} nil)))) (is (spy/called-n-times? scalar/exponential-backoff scalar/RETRIES)) - (is (spy/called-n-times? scalar/prepare-transaction-service! scalar/RETRIES_FOR_RECONNECTION)) - (is (spy/called-n-times? scalar/prepare-storage-service! scalar/RETRIES_FOR_RECONNECTION))))) + (is (spy/called-n-times? scalar/prepare-transaction-service! + (+ (quot scalar/RETRIES scalar/RETRIES_FOR_RECONNECTION) 1))) + (is (spy/called-n-times? scalar/prepare-storage-service! + (+ (quot scalar/RETRIES scalar/RETRIES_FOR_RECONNECTION) 1)))))) (deftest transfer-client-check-tx-test (with-redefs [scalar/check-transaction-states (spy/stub 1)]