467 konserve versions between release and testing are different (#473)

* Add direct dependency on konserve

* Add consistency checking tool for dependencies

* Move non-in-memory and -filesystem backend integration tests out of Datahike into relevant backend projects

* Set internal dependencies

* Drop tools.reader from deps since it doesn't need to be explicitly specified

* Bump minor version number for breaking change of increase in konserve dependency version

* testing CircleCI build trigger

.circleci/config.yml

@@ -110,7 +110,7 @@ workflows:
           context: dockerhub-deploy
           requires:
             - build
-      - tools/integrationtest-postgresql:
+      - tools/integrationtest:
           context: dockerhub-deploy
           requires:
             - build
@@ -126,7 +126,7 @@ workflows:
             - tools/format
             - tools/unittest
             - tools/backward-compatibility-test
-            - tools/integrationtest-postgresql
+            - tools/integrationtest
       - release:
           context:
             - dockerhub-deploy

bin/run-integrationtests

@@ -3,20 +3,4 @@
 set -o errexit
 set -o pipefail
 
-trap teardown EXIT
-
-function setup() {
-    if [ -z ${CIRCLECI} ]; then
-        echo $(docker run --detach --publish 5432:5432 --env POSTGRES_DB=config-test --env POSTGRES_USER=alice --env POSTGRES_PASSWORD=foo postgres:alpine)
-    fi
-}
-
-function teardown() {
-    if [ -z ${CIRCLECI} ]; then
-        docker rm -f ${CONTAINER_NAME}
-    fi
-}
-
-CONTAINER_NAME=$(setup)
-
 TIMBRE_LEVEL=':fatal' bin/kaocha --focus :integration "$@"

build.clj

@@ -5,7 +5,7 @@
     [deps-deploy.deps-deploy :as dd]))
 
 (def lib 'io.replikativ/datahike)
-(def version (format "0.4.%s" (b/git-count-revs nil)))
+(def version (format "0.5.%s" (b/git-count-revs nil)))
 (def current-commit (gh/current-commit))
 (def class-dir "target/classes")
 (def basis (b/create-basis {:project "deps.edn"}))

deps.edn

@@ -1,8 +1,10 @@
 {:deps {org.clojure/clojure                         {:mvn/version "1.11.1"}
         org.clojure/clojurescript                   {:mvn/version "1.11.4"}
+        io.replikativ/hasch                         {:mvn/version "0.3.7"}
         io.replikativ/hitchhiker-tree               {:mvn/version "0.1.11"}
+        io.replikativ/incognito                     {:mvn/version "0.3.66"}
+        io.replikativ/konserve                      {:mvn/version "0.6.0-alpha3"}
         persistent-sorted-set/persistent-sorted-set {:mvn/version "0.1.4"}
-        org.clojure/tools.reader                    {:mvn/version "1.3.6"}
         environ/environ                             {:mvn/version "1.2.0"}
         com.taoensso/timbre                         {:mvn/version "5.2.1"}
         io.replikativ/superv.async                  {:mvn/version "0.3.43"}
@@ -55,6 +57,10 @@
            :deploy {:extra-deps {slipset/deps-deploy {:mvn/version "0.2.0"}}
                     :main-opts ["-m" "deps-deploy.deps-deploy" "deploy" "replikativ-datahike.jar"]}
 
+           :deps-consistency {:extra-paths ["tools"]
+                              :extra-deps {org.clojure/tools.cli {:mvn/version "1.0.206"}}
+                              :main-opts ["-m" "dependencies-consistency-check" "-a" "dev" "-a" "test"]}
+
            :install {:extra-deps {slipset/deps-deploy {:mvn/version "0.2.0"}}
                      :exec-fn deps-deploy.deps-deploy/deploy
                      :exec-args {:installer :local

tools/dependencies_consistency_check.clj

@@ -0,0 +1,84 @@
+(ns dependencies-consistency-check
+  (:require [clojure.edn :as edn]
+            [clojure.java.io :as io]
+            [clojure.java.shell :as sh]
+            [clojure.pprint :as pp]
+            [clojure.set :as set]
+            [clojure.string :as string]
+            [clojure.tools.cli :as cli]))
+
+(defn read-edn [filepath]
+  (edn/read (java.io.PushbackReader. (io/reader filepath))))
+
+(def project-aliases (:aliases (read-edn "deps.edn")))
+
+(def cli-options
+  [;; Alias(es) to include in check
+   ["-a" "--alias ALIAS" "Alias (without \":\", which will be prefixed onto argument)"
+    :parse-fn #(keyword %)
+    :multi true
+    :update-fn conj
+    :validate [#(% project-aliases) #(str "Invalid alias: " % " is undeclared in deps.edn")]]
+   ["-A" "--aliases-only" "Exclude direct (release) dependencies"]
+   ["-o" "--output FILEPATH" "Filepath to which output should be written"
+    :default nil]
+   ["-h" "--help"]])
+
+(defn exit [status msg]
+  (println msg)
+  (System/exit status))
+
+(defn usage [summary]
+  (string/join "\n" ["Usage: clj -M:deps-consistency [options]\n" "Options:" summary]))
+
+(defn read-trace [trace-file]
+  (read-edn trace-file))
+
+(defn create-trace-file [alias filename]
+  (sh/sh "clj" (str "-A" (str alias)) "-Strace")
+  (when (not= "trace.edn" filename) (sh/sh "mv" "trace.edn" filename)))
+
+(defn get-selected-versions [trace]
+  (let [vmap (:vmap trace)]
+    (zipmap (keys vmap) (map :select (vals vmap)))))
+
+(defn map-build-pkg-vns [alias-filenames]
+  (zipmap (keys alias-filenames)
+          (map #(get-selected-versions (read-trace %)) (vals alias-filenames))))
+
+(defn map-pkg-build-vns [build-pkg-vns]
+  (let [all-pkgs (apply set/union (map keys (vals build-pkg-vns)))]
+    (persistent! (reduce-kv (fn [outer-map build pkg-vns]
+                              (reduce-kv (fn [build-map pkg vn]
+                                           (assoc! build-map pkg (assoc (pkg build-map) build vn)))
+                                         outer-map pkg-vns))
+                            (transient (zipmap all-pkgs (repeat {}))) build-pkg-vns))))
+
+(defn get-inconsistent-pkgs [pkg-build-versions]
+  (persistent! (reduce-kv (fn [m pkg build-vns]
+                            (if (> (count (distinct (vals build-vns))) 1)
+                              (assoc! m pkg build-vns) m))
+                          (transient {}) pkg-build-versions)))
+
+(defn -main [& args]
+  (let [{:keys [options errors summary] :as parsed-opts} (cli/parse-opts args cli-options)
+        {:keys [alias aliases-only output]} options
+        tmpdir (System/getProperty "java.io.tmpdir")
+        trace-filenames #(str tmpdir "trace-" (name %) ".edn")
+        alias-filenames (cond-> (zipmap alias (map trace-filenames alias))
+                          (nil? aliases-only) (assoc :release (str tmpdir "trace.edn")))]
+    (cond
+      (:help options)
+      (exit 0 (usage summary))
+
+      errors
+      (exit 1 (string/join "\n" ["Errors:" (string/join "\n" errors) "" (usage summary)]))
+
+      :else
+      (do (doseq [[alias filename] alias-filenames] (create-trace-file alias filename))
+          (pp/pprint (-> alias-filenames
+                         map-build-pkg-vns
+                         map-pkg-build-vns
+                         get-inconsistent-pkgs)
+                     (if output (io/writer output) *out*))
+          (apply sh/sh "rm" (vals alias-filenames))))))