From 82acdb0e6928d55f61e57e4a7a65ddf7e8b055fe Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Thu, 24 Oct 2024 22:06:31 +0900 Subject: [PATCH 1/4] Add upper limit to `exponential-backoff` --- scalardb/src/scalardb/core.clj | 7 ++++++- scalardb/test/scalardb/core_test.clj | 6 ++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/scalardb/src/scalardb/core.clj b/scalardb/src/scalardb/core.clj index 2ebd860..063a20b 100644 --- a/scalardb/src/scalardb/core.clj +++ b/scalardb/src/scalardb/core.clj @@ -15,15 +15,20 @@ (def ^:const RETRIES 8) (def ^:const RETRIES_FOR_RECONNECTION 3) (def ^:private ^:const NUM_FAILURES_FOR_RECONNECTION 1000) +(def ^:private ^:const MAX_WAIT_MILLIS 32000) (def ^:const INITIAL_TABLE_ID 1) (def ^:const DEFAULT_TABLE_COUNT 3) (def ^:const KEYSPACE "jepsen") (def ^:const VERSION "tx_version") +(defn compute-exponential-backoff + [r] + (min MAX_WAIT_MILLIS (reduce * 1000 (repeat r 2)))) + (defn exponential-backoff [r] - (Thread/sleep (reduce * 1000 (repeat r 2)))) + (Thread/sleep (compute-exponential-backoff r))) (defn- get-cassandra-schema "Only the current test schemata are covered diff --git a/scalardb/test/scalardb/core_test.clj b/scalardb/test/scalardb/core_test.clj index 00d00de..1545bac 100644 --- a/scalardb/test/scalardb/core_test.clj +++ b/scalardb/test/scalardb/core_test.clj @@ -152,3 +152,9 @@ (scalar/check-transaction-states test #{"tx"}))) (is (spy/called-n-times? scalar/exponential-backoff 8)) (is (spy/called-n-times? scalar/prepare-storage-service! 3))))) + +(deftest compute-exponential-backoff-test + (is (= 2000 (scalar/compute-exponential-backoff 1))) + (is (= 4000 (scalar/compute-exponential-backoff 2))) + (is (= 32000 (scalar/compute-exponential-backoff 5))) + (is (= 32000 (scalar/compute-exponential-backoff 10)))) From 78f9f2eb95bd6747b48357fdb672f493fd611b02 Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Fri, 25 Oct 2024 10:58:13 +0900 Subject: [PATCH 2/4] Increase max retry --- scalardb/src/scalardb/core.clj | 2 +- scalardb/test/scalardb/core_test.clj | 6 +++--- scalardb/test/scalardb/transfer_2pc_test.clj | 6 ++++-- scalardb/test/scalardb/transfer_append_2pc_test.clj | 3 ++- scalardb/test/scalardb/transfer_append_test.clj | 3 ++- scalardb/test/scalardb/transfer_test.clj | 6 ++++-- 6 files changed, 16 insertions(+), 10 deletions(-) diff --git a/scalardb/src/scalardb/core.clj b/scalardb/src/scalardb/core.clj index 063a20b..9f77f0b 100644 --- a/scalardb/src/scalardb/core.clj +++ b/scalardb/src/scalardb/core.clj @@ -12,7 +12,7 @@ StorageFactory) (com.scalar.db.transaction.consensuscommit Coordinator))) -(def ^:const RETRIES 8) +(def ^:const RETRIES 20) (def ^:const RETRIES_FOR_RECONNECTION 3) (def ^:private ^:const NUM_FAILURES_FOR_RECONNECTION 1000) (def ^:private ^:const MAX_WAIT_MILLIS 32000) diff --git a/scalardb/test/scalardb/core_test.clj b/scalardb/test/scalardb/core_test.clj index 1545bac..0bccaec 100644 --- a/scalardb/test/scalardb/core_test.clj +++ b/scalardb/test/scalardb/core_test.clj @@ -103,7 +103,7 @@ scalar/create-service-instance (spy/stub nil)] (let [test {:db mock-db :storage (atom nil)}] (scalar/prepare-storage-service! test) - (is (spy/called-n-times? scalar/exponential-backoff 8)) + (is (spy/called-n-times? scalar/exponential-backoff 20)) (is (nil? @(:storage test)))))) (deftest check-connection-test @@ -150,8 +150,8 @@ (let [test {:storage (atom mock-storage)}] (is (thrown? clojure.lang.ExceptionInfo (scalar/check-transaction-states test #{"tx"}))) - (is (spy/called-n-times? scalar/exponential-backoff 8)) - (is (spy/called-n-times? scalar/prepare-storage-service! 3))))) + (is (spy/called-n-times? scalar/exponential-backoff 20)) + (is (spy/called-n-times? scalar/prepare-storage-service! 7))))) (deftest compute-exponential-backoff-test (is (= 2000 (scalar/compute-exponential-backoff 1))) diff --git a/scalardb/test/scalardb/transfer_2pc_test.clj b/scalardb/test/scalardb/transfer_2pc_test.clj index 09a82f6..95e3301 100644 --- a/scalardb/test/scalardb/transfer_2pc_test.clj +++ b/scalardb/test/scalardb/transfer_2pc_test.clj @@ -235,8 +235,10 @@ (#'transfer/get-all {:client client} nil)))) (is (spy/called-n-times? scalar/exponential-backoff scalar/RETRIES)) - (is (spy/called-n-times? scalar/prepare-transaction-service! scalar/RETRIES_FOR_RECONNECTION)) - (is (spy/called-n-times? scalar/prepare-storage-service! scalar/RETRIES_FOR_RECONNECTION))))) + (is (spy/called-n-times? scalar/prepare-transaction-service! + (+ (quot scalar/RETRIES scalar/RETRIES_FOR_RECONNECTION) 1)))) + (is (spy/called-n-times? scalar/prepare-storage-service! + (+ (quot scalar/RETRIES scalar/RETRIES_FOR_RECONNECTION) 1))))) (deftest transfer-client-check-tx-test (with-redefs [scalar/check-transaction-states (spy/stub 1)] diff --git a/scalardb/test/scalardb/transfer_append_2pc_test.clj b/scalardb/test/scalardb/transfer_append_2pc_test.clj index b4db93f..c7f0844 100644 --- a/scalardb/test/scalardb/transfer_append_2pc_test.clj +++ b/scalardb/test/scalardb/transfer_append_2pc_test.clj @@ -242,7 +242,8 @@ (client/invoke! client {:db mock-db} (transfer/get-all {:client client} nil)))) (is (spy/called-n-times? scalar/exponential-backoff scalar/RETRIES)) - (is (spy/called-n-times? scalar/prepare-transaction-service! scalar/RETRIES_FOR_RECONNECTION))))) + (is (spy/called-n-times? scalar/prepare-transaction-service! + (+ (quot scalar/RETRIES scalar/RETRIES_FOR_RECONNECTION) 1)))))) (deftest transfer-client-check-tx-test (with-redefs [scalar/check-transaction-states (spy/stub 1)] diff --git a/scalardb/test/scalardb/transfer_append_test.clj b/scalardb/test/scalardb/transfer_append_test.clj index 8ef72d0..a265b2c 100644 --- a/scalardb/test/scalardb/transfer_append_test.clj +++ b/scalardb/test/scalardb/transfer_append_test.clj @@ -212,7 +212,8 @@ (client/invoke! client {:db mock-db} (transfer/get-all {:client client} nil)))) (is (spy/called-n-times? scalar/exponential-backoff scalar/RETRIES)) - (is (spy/called-n-times? scalar/prepare-transaction-service! scalar/RETRIES_FOR_RECONNECTION))))) + (is (spy/called-n-times? scalar/prepare-transaction-service! + (+ (quot scalar/RETRIES scalar/RETRIES_FOR_RECONNECTION) 1)))))) (deftest transfer-client-check-tx-test (with-redefs [scalar/check-transaction-states (spy/stub 1)] diff --git a/scalardb/test/scalardb/transfer_test.clj b/scalardb/test/scalardb/transfer_test.clj index f4a9cb2..ac0d664 100644 --- a/scalardb/test/scalardb/transfer_test.clj +++ b/scalardb/test/scalardb/transfer_test.clj @@ -205,8 +205,10 @@ (#'transfer/get-all {:client client} nil)))) (is (spy/called-n-times? scalar/exponential-backoff scalar/RETRIES)) - (is (spy/called-n-times? scalar/prepare-transaction-service! scalar/RETRIES_FOR_RECONNECTION)) - (is (spy/called-n-times? scalar/prepare-storage-service! scalar/RETRIES_FOR_RECONNECTION))))) + (is (spy/called-n-times? scalar/prepare-transaction-service! + (+ (quot scalar/RETRIES scalar/RETRIES_FOR_RECONNECTION) 1)))) + (is (spy/called-n-times? scalar/prepare-storage-service! + (+ (quot scalar/RETRIES scalar/RETRIES_FOR_RECONNECTION) 1))))) (deftest transfer-client-check-tx-test (with-redefs [scalar/check-transaction-states (spy/stub 1)] From ce9087fa8d1328100851051aa54f27433f0eadf9 Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Fri, 25 Oct 2024 11:44:24 +0900 Subject: [PATCH 3/4] Fix unit test --- scalardb/test/scalardb/transfer_2pc_test.clj | 4 ++-- scalardb/test/scalardb/transfer_test.clj | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scalardb/test/scalardb/transfer_2pc_test.clj b/scalardb/test/scalardb/transfer_2pc_test.clj index 95e3301..6ad953b 100644 --- a/scalardb/test/scalardb/transfer_2pc_test.clj +++ b/scalardb/test/scalardb/transfer_2pc_test.clj @@ -236,9 +236,9 @@ nil)))) (is (spy/called-n-times? scalar/exponential-backoff scalar/RETRIES)) (is (spy/called-n-times? scalar/prepare-transaction-service! - (+ (quot scalar/RETRIES scalar/RETRIES_FOR_RECONNECTION) 1)))) + (+ (quot scalar/RETRIES scalar/RETRIES_FOR_RECONNECTION) 1))) (is (spy/called-n-times? scalar/prepare-storage-service! - (+ (quot scalar/RETRIES scalar/RETRIES_FOR_RECONNECTION) 1))))) + (+ (quot scalar/RETRIES scalar/RETRIES_FOR_RECONNECTION) 1)))))) (deftest transfer-client-check-tx-test (with-redefs [scalar/check-transaction-states (spy/stub 1)] diff --git a/scalardb/test/scalardb/transfer_test.clj b/scalardb/test/scalardb/transfer_test.clj index ac0d664..b750c60 100644 --- a/scalardb/test/scalardb/transfer_test.clj +++ b/scalardb/test/scalardb/transfer_test.clj @@ -206,9 +206,9 @@ nil)))) (is (spy/called-n-times? scalar/exponential-backoff scalar/RETRIES)) (is (spy/called-n-times? scalar/prepare-transaction-service! - (+ (quot scalar/RETRIES scalar/RETRIES_FOR_RECONNECTION) 1)))) + (+ (quot scalar/RETRIES scalar/RETRIES_FOR_RECONNECTION) 1))) (is (spy/called-n-times? scalar/prepare-storage-service! - (+ (quot scalar/RETRIES scalar/RETRIES_FOR_RECONNECTION) 1))))) + (+ (quot scalar/RETRIES scalar/RETRIES_FOR_RECONNECTION) 1)))))) (deftest transfer-client-check-tx-test (with-redefs [scalar/check-transaction-states (spy/stub 1)] From dec8f46b91552199ab40aafd9818729212e7cc9e Mon Sep 17 00:00:00 2001 From: Mitsunori Komatsu Date: Tue, 29 Oct 2024 11:03:22 +0900 Subject: [PATCH 4/4] Use the constants in core_test.cli --- scalardb/test/scalardb/core_test.clj | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scalardb/test/scalardb/core_test.clj b/scalardb/test/scalardb/core_test.clj index 0bccaec..1ee27b2 100644 --- a/scalardb/test/scalardb/core_test.clj +++ b/scalardb/test/scalardb/core_test.clj @@ -103,7 +103,7 @@ scalar/create-service-instance (spy/stub nil)] (let [test {:db mock-db :storage (atom nil)}] (scalar/prepare-storage-service! test) - (is (spy/called-n-times? scalar/exponential-backoff 20)) + (is (spy/called-n-times? scalar/exponential-backoff scalardb.core/RETRIES)) (is (nil? @(:storage test)))))) (deftest check-connection-test @@ -150,8 +150,9 @@ (let [test {:storage (atom mock-storage)}] (is (thrown? clojure.lang.ExceptionInfo (scalar/check-transaction-states test #{"tx"}))) - (is (spy/called-n-times? scalar/exponential-backoff 20)) - (is (spy/called-n-times? scalar/prepare-storage-service! 7))))) + (is (spy/called-n-times? scalar/exponential-backoff scalardb.core/RETRIES)) + (is (spy/called-n-times? scalar/prepare-storage-service! + (+ (quot scalar/RETRIES scalar/RETRIES_FOR_RECONNECTION) 1)))))) (deftest compute-exponential-backoff-test (is (= 2000 (scalar/compute-exponential-backoff 1)))