Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve retry for scalardb test #131

Merged
merged 4 commits into from
Oct 29, 2024
Merged

Improve retry for scalardb test #131

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
82acdb0
Add upper limit to `exponential-backoff`
komamitsu Oct 24, 2024
78f9f2e
Increase max retry
komamitsu Oct 25, 2024
ce9087f
Fix unit test
komamitsu Oct 25, 2024
dec8f46
Use the constants in core_test.cli
komamitsu Oct 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions scalardb/src/scalardb/core.clj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,18 +12,23 @@
StorageFactory)
(com.scalar.db.transaction.consensuscommit Coordinator)))

(def ^:const RETRIES 8)
(def ^:const RETRIES 20)
(def ^:const RETRIES_FOR_RECONNECTION 3)
(def ^:private ^:const NUM_FAILURES_FOR_RECONNECTION 1000)
(def ^:private ^:const MAX_WAIT_MILLIS 32000)
(def ^:const INITIAL_TABLE_ID 1)
(def ^:const DEFAULT_TABLE_COUNT 3)

(def ^:const KEYSPACE "jepsen")
(def ^:const VERSION "tx_version")

(defn compute-exponential-backoff
[r]
(min MAX_WAIT_MILLIS (reduce * 1000 (repeat r 2))))

(defn exponential-backoff
[r]
(Thread/sleep (reduce * 1000 (repeat r 2))))
(Thread/sleep (compute-exponential-backoff r)))

(defn- get-cassandra-schema
"Only the current test schemata are covered
Expand Down
13 changes: 10 additions & 3 deletions scalardb/test/scalardb/core_test.clj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@
scalar/create-service-instance (spy/stub nil)]
(let [test {:db mock-db :storage (atom nil)}]
(scalar/prepare-storage-service! test)
(is (spy/called-n-times? scalar/exponential-backoff 8))
(is (spy/called-n-times? scalar/exponential-backoff scalardb.core/RETRIES))
(is (nil? @(:storage test))))))

(deftest check-connection-test
Expand Down Expand Up @@ -150,5 +150,12 @@
(let [test {:storage (atom mock-storage)}]
(is (thrown? clojure.lang.ExceptionInfo
(scalar/check-transaction-states test #{"tx"})))
(is (spy/called-n-times? scalar/exponential-backoff 8))
(is (spy/called-n-times? scalar/prepare-storage-service! 3)))))
(is (spy/called-n-times? scalar/exponential-backoff scalardb.core/RETRIES))
(is (spy/called-n-times? scalar/prepare-storage-service!
(+ (quot scalar/RETRIES scalar/RETRIES_FOR_RECONNECTION) 1))))))

(deftest compute-exponential-backoff-test
(is (= 2000 (scalar/compute-exponential-backoff 1)))
(is (= 4000 (scalar/compute-exponential-backoff 2)))
(is (= 32000 (scalar/compute-exponential-backoff 5)))
(is (= 32000 (scalar/compute-exponential-backoff 10))))
6 changes: 4 additions & 2 deletions scalardb/test/scalardb/transfer_2pc_test.clj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -235,8 +235,10 @@
(#'transfer/get-all {:client client}
nil))))
(is (spy/called-n-times? scalar/exponential-backoff scalar/RETRIES))
(is (spy/called-n-times? scalar/prepare-transaction-service! scalar/RETRIES_FOR_RECONNECTION))
(is (spy/called-n-times? scalar/prepare-storage-service! scalar/RETRIES_FOR_RECONNECTION)))))
(is (spy/called-n-times? scalar/prepare-transaction-service!
(+ (quot scalar/RETRIES scalar/RETRIES_FOR_RECONNECTION) 1)))
(is (spy/called-n-times? scalar/prepare-storage-service!
(+ (quot scalar/RETRIES scalar/RETRIES_FOR_RECONNECTION) 1))))))

(deftest transfer-client-check-tx-test
(with-redefs [scalar/check-transaction-states (spy/stub 1)]
Expand Down
3 changes: 2 additions & 1 deletion scalardb/test/scalardb/transfer_append_2pc_test.clj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -242,7 +242,8 @@
(client/invoke! client {:db mock-db}
(transfer/get-all {:client client} nil))))
(is (spy/called-n-times? scalar/exponential-backoff scalar/RETRIES))
(is (spy/called-n-times? scalar/prepare-transaction-service! scalar/RETRIES_FOR_RECONNECTION)))))
(is (spy/called-n-times? scalar/prepare-transaction-service!
(+ (quot scalar/RETRIES scalar/RETRIES_FOR_RECONNECTION) 1))))))

(deftest transfer-client-check-tx-test
(with-redefs [scalar/check-transaction-states (spy/stub 1)]
Expand Down
3 changes: 2 additions & 1 deletion scalardb/test/scalardb/transfer_append_test.clj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -212,7 +212,8 @@
(client/invoke! client {:db mock-db}
(transfer/get-all {:client client} nil))))
(is (spy/called-n-times? scalar/exponential-backoff scalar/RETRIES))
(is (spy/called-n-times? scalar/prepare-transaction-service! scalar/RETRIES_FOR_RECONNECTION)))))
(is (spy/called-n-times? scalar/prepare-transaction-service!
(+ (quot scalar/RETRIES scalar/RETRIES_FOR_RECONNECTION) 1))))))

(deftest transfer-client-check-tx-test
(with-redefs [scalar/check-transaction-states (spy/stub 1)]
Expand Down
6 changes: 4 additions & 2 deletions scalardb/test/scalardb/transfer_test.clj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -205,8 +205,10 @@
(#'transfer/get-all {:client client}
nil))))
(is (spy/called-n-times? scalar/exponential-backoff scalar/RETRIES))
(is (spy/called-n-times? scalar/prepare-transaction-service! scalar/RETRIES_FOR_RECONNECTION))
(is (spy/called-n-times? scalar/prepare-storage-service! scalar/RETRIES_FOR_RECONNECTION)))))
(is (spy/called-n-times? scalar/prepare-transaction-service!
(+ (quot scalar/RETRIES scalar/RETRIES_FOR_RECONNECTION) 1)))
(is (spy/called-n-times? scalar/prepare-storage-service!
(+ (quot scalar/RETRIES scalar/RETRIES_FOR_RECONNECTION) 1))))))

(deftest transfer-client-check-tx-test
(with-redefs [scalar/check-transaction-states (spy/stub 1)]
Expand Down
Loading