fix nodes

cassandra/src/cassandra/nemesis.clj

@@ -199,15 +199,15 @@
 
 (defn flush-generator
   [opts]
-  (when (contains? (:admin opts) :flush-compact)
+  (when (contains? (:admin opts) :flush)
     (->> (gen/mix [(repeat {:type :info, :f :flush})
                    (repeat {:type :info, :f :compact})])
          (gen/stagger default-interval))))
 
 (defn flush-package
   "A combined nemesis package for flush and compaction."
   [opts]
-  (when (contains? (:admin opts) :flush-compact)
+  (when (contains? (:admin opts) :flush)
     {:nemesis   (flush-nemesis)
      :generator (flush-generator opts)
      :perf      #{{:name  "flush"

cassandra/src/cassandra/runner.clj

@@ -42,7 +42,7 @@
 (def admin
   {"none" []
    "join" [:join]
-   "flush" [:flush-compact]})
+   "flush" [:flush]})
 
 (def test-opt-spec
   [(cli/repeated-opt nil "--workload NAME" "Test(s) to run" [] workload-keys)])

cassandra/test/cassandra/runner_test.clj

@@ -6,7 +6,7 @@
   (let [opts {:target "cassandra"
               :workload :batch
               :nemesis [:crash]
-              :admin [:flush-compact]
+              :admin [:flush]
               :time-limit 60}
         test (runner/cassandra-test opts)]
-    (is (= "cassandra-batch-crash-flush-compact" (:name test)))))
+    (is (= "cassandra-batch-crash-flush" (:name test)))))

scalardb/src/scalardb/db/postgres.clj

@@ -8,6 +8,8 @@
             [jepsen.os.debian :as debian]))
 
 (def ^:private ^:const DEFAULT_VERSION "15")
+(def ^:private ^:const TIMEOUT_SEC 600)
+(def ^:private ^:const INTERVAL_SEC 10)
 
 (defn- install!
   "Installs PostgreSQL."
@@ -69,6 +71,28 @@
   (c/su (meh (c/exec :rm :-r (get-main-dir version))))
   (c/su (meh (c/exec :rm (get-log-path version)))))
 
+(defn live-node?
+  [test]
+  (let [node (-> test :nodes first)]
+    (try
+      (c/on node (c/sudo "postgres" (c/exec :pg_isready)))
+      true
+      (catch Exception _
+        (info node "is down")
+        false))))
+
+(defn wait-for-recovery
+  "Wait for the node bootstrapping."
+  ([test]
+   (wait-for-recovery TIMEOUT_SEC INTERVAL_SEC test))
+  ([timeout-sec interval-sec test]
+   (when-not (live-node? test)
+     (Thread/sleep (* interval-sec 1000))
+     (if (>= timeout-sec interval-sec)
+       (wait-for-recovery (- timeout-sec interval-sec) interval-sec test)
+       (throw (ex-info "Timed out waiting for the postgres node"
+                       {:cause "The node couldn't start"}))))))
+
 (defn db
   "Setup PostgreSQL."
   []

scalardb/src/scalardb/db_extend.clj

@@ -1,7 +1,8 @@
 (ns scalardb.db-extend
   (:require [cassandra.core :as cassandra]
             [clojure.string :as string]
-            [jepsen.db :as db])
+            [jepsen.db :as db]
+            [scalardb.db.postgres :as postgres])
   (:import (com.scalar.db.storage.cassandra CassandraAdmin
                                             CassandraAdmin$ReplicationStrategy
                                             CassandraAdmin$CompactionStrategy)
@@ -47,15 +48,13 @@
 
 (defrecord ExtPostgres []
   DbExtension
-  ;; TODO: check the liveness
-  (live-nodes [_ test] (:nodes test))
-  (wait-for-recovery [_ _])
+  (live-nodes [_ test] (postgres/live-node? test))
+  (wait-for-recovery [_ test] (postgres/wait-for-recovery test))
   (create-table-opts [_ _] {})
   (create-properties
-    [this test]
-    (let [node (first (live-nodes this test))]
-      (when (nil? node)
-        (throw (ex-info "No living node" {:test test})))
+    [_ test]
+    (let [node (-> test :nodes first)]
+      ;; We have only one node in this test
       (doto (Properties.)
         (.setProperty "scalar.db.storage" "jdbc")
         (.setProperty "scalar.db.contact_points"

scalardb/src/scalardb/runner.clj

@@ -34,7 +34,13 @@
   "Returns [extended-db constructed-nemesis num-max-nodes]."
   [db-key faults admin]
   (case db-key
-    :cassandra (let [db (extend-db (cassandra/db) :cassandra)]
+    :cassandra (let [db (extend-db (cassandra/db) :cassandra)
+                     ;; replace :kill nemesis with :crash for Cassandra
+                     faults (mapv #(if (= % :kill) :crash %) faults)]
+                 (when-not (every? #(some? (get cr/nemeses (name %))) faults)
+                   (throw
+                    (ex-info
+                     (str "Invalid nemesis for Cassandra: " faults) {})))
                  [db
                   (cn/nemesis-package
                    {:db db
@@ -47,6 +53,8 @@
                                           :minority-third]}})
                   Integer/MAX_VALUE])
     :postgres (let [db (extend-db (postgres/db) :postgres)]
+                (when (seq admin)
+                  (warn "The admin operations are ignored:" admin))
                 [db
                  (jn/nemesis-package
                   {:db db
@@ -87,8 +95,7 @@
    "packet"    [:packet]
    "clock"     [:clock]
    "crash"     [:kill]
-   "pause"     [:pause]
-   "mix"       [:kill :partition :clock]})
+   "pause"     [:pause]})
 
 (def test-opt-spec
   [(cli/repeated-opt nil "--db NAME" "DB(s) on which the test is run"
@@ -124,18 +131,6 @@
       (into (map name admin))
       (->> (remove nil?) (string/join "-"))))
 
-(defn- validate-options
-  "Check if the given faults are valid for the tested DB."
-  [db nemesis admin]
-  (when (= db :cassandra)
-    (when-not (every? #(some? (% cr/nemeses)) nemesis)
-      (throw (ex-info (str "Invalid nemesis for Cassandra: " nemesis) {})))
-    (when-not (every? #(some? (% cr/admin)) admin)
-      (throw (ex-info (str "Invalid admin for Cassandra: " admin) {}))))
-  (when (not= db :cassandra)
-    (when (seq admin)
-      (warn "The admin operations are ignored:" admin))))
-
 (def ^:private scalardb-opts
   {:storage (atom nil)
    :transaction (atom nil)
@@ -151,7 +146,7 @@
         consistency-model (->> base-opts :consistency-model (mapv keyword))
         workload-opts (merge base-opts
                              scalardb-opts
-                             {:nodes (take max-nodes (:nodes base-opts))
+                             {:nodes (vec (take max-nodes (:nodes base-opts)))
                               :consistency-model consistency-model})
         workload ((workload-key workloads) workload-opts)]
     (merge tests/noop-test
@@ -186,7 +181,6 @@
                           workload-key (:workload options)
                           faults (:nemesis options)
                           admin (:admin options)]
-                    (validate-options db-key faults admin)
                     (let [test (-> options
                                    (scalardb-test db-key
                                                   workload-key