Skip to content

Commit

Permalink
Merge pull request #297 from replikativ/move-0.3.4-to-master
Browse files Browse the repository at this point in the history 
Move 0.3.4 to master
  • Loading branch information
@grischoun
grischoun authored Apr 2, 2021
2 parents 6b93d78 + 543aceb commit 287124e
Show file tree
Hide file tree
Showing 27 changed files with 500 additions and 459 deletions.
3 changes: 3 additions & 0 deletions .github/FUNDING.yml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# These are supported funding model platforms

github: replikativ
2 changes: 2 additions & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,3 +23,5 @@ release.clj
lsp/
.calva/
out/
.clj-kondo
.lsp
1 change: 0 additions & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
<a href="https://clojars.org/io.replikativ/datahike"> <img src="https://img.shields.io/clojars/v/io.replikativ/datahike.svg" /></a>
<a href="https://circleci.com/gh/replikativ/datahike"><img src="https://circleci.com/gh/replikativ/datahike.svg?style=shield"/></a>
<a href="https://github.com/replikativ/datahike/tree/development"><img src="https://img.shields.io/github/last-commit/replikativ/datahike/development"/></a>
<a href="https://versions.deps.co/replikativ/datahike" title="Dependencies Status"><img src="https://versions.deps.co/replikativ/datahike/status.svg" /></a>
</p>

Datahike is a durable [Datalog](https://en.wikipedia.org/wiki/Datalog) database
Expand Down
13 changes: 11 additions & 2 deletions benchmark/src/benchmark/core.clj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,16 @@
(and (some (comp not nil?) server-description)
(not (full-server-description? server-description))))

(defn time-statistics [times]
(let [mean (/ (apply + times) (count times))
std (->> times
(map #(* (- % mean) (- % mean)))
(apply +)
(* (/ 1.0 (count times)))
Math/sqrt)]
{:mean-time mean
:std-time std}))

(defn -main [& args]
(let [{:keys [options errors summary]} (cli/parse-opts args cli-options)
server-info-keys [:db-server-url :db-token :db-name]
Expand Down Expand Up @@ -57,8 +67,7 @@
(apply concat)
(group-by :context)
(map (fn [[context group]]
(assoc context :mean-time (/ (reduce (fn [x y] (+ x (:time y))) 0 group)
(count group))))))
(merge context (time-statistics (map :time group))))))
tagged (if (empty? tags)
(vec processed)
(mapv (fn [entity] (assoc entity :tag (vec tags))) processed))]
Expand Down
5 changes: 4 additions & 1 deletion bin/run-unittests
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
#!/usr/bin/env bash

TIMBRE_LEVEL=':fatal' ./bin/kaocha --focus :clj
set -o errexit
set -o pipefail

TIMBRE_LEVEL=':fatal' ./bin/kaocha --focus :clj $@
9 changes: 7 additions & 2 deletions deps.edn
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@
org.clojure/tools.reader {:mvn/version "1.3.3"}
environ/environ {:mvn/version "1.2.0"}
com.taoensso/timbre {:mvn/version "5.1.0"}
io.replikativ/hitchhiker-tree {:mvn/version "0.1.7"}
;;io.replikativ/hitchhiker-tree {:mvn/version "0.1.7"}
io.replikativ/hitchhiker-tree {:git/url "https://github.com/replikativ/hitchhiker-tree.git"
:sha "4d332a2a00d460ec59ab8cc5468eb7272e99fe3e"}
io.replikativ/superv.async {:mvn/version "0.2.11"}
io.lambdaforge/datalog-parser {:mvn/version "0.1.8"}
io.replikativ/zufall {:mvn/version "0.1.0"}
Expand Down Expand Up @@ -49,4 +51,7 @@
:main-opts ["-m" "hf.depstar.jar" "replikativ-datahike.jar"]}

:format {:extra-deps {cljfmt/cljfmt {:mvn/version "0.7.0"}}
:main-opts ["-m" "cljfmt.main" "check"]}}}
:main-opts ["-m" "cljfmt.main" "check"]}

:ffix {:extra-deps {cljfmt/cljfmt {:mvn/version "0.7.0"}}
:main-opts ["-m" "cljfmt.main" "fix"]}}}
8 changes: 6 additions & 2 deletions doc/contributing.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,13 @@
clj -M:repl
```

## Running the tests
- `./bin/run-unittests` or `./bin/run-unittests --watch`
- `./bin/run-integrationtests` (Docker needed)

## Starting the benchmark
```
clj -M:benchmark
TIMBRE_LEVEL=':info' clj -M:benchmark
```

## Building a Datahike jar
Expand All @@ -23,7 +27,7 @@ clj -M:format
```
or fix the formatting:
```
clj -Sdeps '{:deps {cljfmt/cljfmt {:mvn/version "0.7.0"}}}' -m cljfmt.main fix
clj -M:ffix
```

## Releasing Datahike
Expand Down
2 changes: 1 addition & 1 deletion pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
<groupId>io.replikativ</groupId>
<artifactId>datahike</artifactId>
<packaging>jar</packaging>
<version>0.3.3</version>
<version>0.3.4</version>
<name>datahike</name>
<description>A durable datalog implementation adaptable for distribution.</description>
<url>https://github.com/replikativ/datahike</url>
Expand Down
10 changes: 7 additions & 3 deletions src/datahike/connector.cljc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

(defn update-and-flush-db [connection tx-data update-fn]
(let [{:keys [db-after] :as tx-report} @(update-fn connection tx-data)
{:keys [eavt aevt avet temporal-eavt temporal-aevt temporal-avet schema rschema config max-tx hash]} db-after
{:keys [eavt aevt avet temporal-eavt temporal-aevt temporal-avet schema rschema config max-tx op-count hash]} db-after
store (:store @connection)
backend (kons/->KonserveBackend store)
eavt-flushed (di/-flush eavt backend)
Expand All @@ -36,6 +36,7 @@
:config config
:hash hash
:max-tx max-tx
:op-count op-count
:eavt-key eavt-flushed
:aevt-key aevt-flushed
:avet-key avet-flushed}
Expand Down Expand Up @@ -139,7 +140,8 @@
(ds/release-store store-config store)
(dt/raise "Database does not exist." {:type :db-does-not-exist
:config config}))
{:keys [eavt-key aevt-key avet-key temporal-eavt-key temporal-aevt-key temporal-avet-key schema rschema config max-tx hash]} stored-db
{:keys [eavt-key aevt-key avet-key temporal-eavt-key temporal-aevt-key temporal-avet-key schema rschema config max-tx op-count hash]
:or {op-count 0}} stored-db
empty (db/empty-db nil config)]
(d/conn-from-db
(assoc empty
Expand All @@ -148,6 +150,7 @@
:schema schema
:hash hash
:max-eid (db/init-max-eid eavt-key)
:op-count op-count
:eavt eavt-key
:aevt aevt-key
:avet avet-key
Expand All @@ -166,12 +169,13 @@
stored-db (<?? S (k/get-in store [:db]))
_ (when stored-db
(dt/raise "Database already exists." {:type :db-already-exists :config store-config}))
{:keys [eavt aevt avet temporal-eavt temporal-aevt temporal-avet schema rschema config max-tx hash]}
{:keys [eavt aevt avet temporal-eavt temporal-aevt temporal-avet schema rschema config max-tx op-count hash]}
(db/empty-db nil config)
backend (kons/->KonserveBackend store)]
(<?? S (k/assoc-in store [:db]
(merge {:schema schema
:max-tx max-tx
:op-count op-count
:hash hash
:rschema rschema
:config config
Expand Down
81 changes: 46 additions & 35 deletions src/datahike/db.cljc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -195,7 +195,7 @@
(filter (fn [^Datom d] (= tx (datom-tx d))) (-all eavt)) ;; _ _ _ tx
(-all eavt)]))))

(defrecord-updatable DB [schema eavt aevt avet temporal-eavt temporal-aevt temporal-avet max-eid max-tx rschema hash config]
(defrecord-updatable DB [schema eavt aevt avet temporal-eavt temporal-aevt temporal-avet max-eid max-tx op-count rschema hash config]
#?@(:cljs
[IHash (-hash [db] hash)
IEquiv (-equiv [db other] (equiv-db db other))
Expand Down Expand Up @@ -818,7 +818,8 @@
:avet (di/empty-index index :avet)
:max-eid e0
:max-tx tx0
:hash 0}
:hash 0
:op-count 0}
(when keep-history?
{:temporal-eavt (di/empty-index index :eavt)
:temporal-aevt (di/empty-index index :aevt)
Expand All @@ -845,11 +846,13 @@
(let [{:keys [index schema-flexibility keep-history?] :as config} (merge (dc/storeless-config) config)
rschema (rschema (merge implicit-schema schema))
indexed (:db/index rschema)
eavt (di/init-index index datoms indexed :eavt)
aevt (di/init-index index datoms indexed :aevt)
avet (di/init-index index datoms indexed :avet)
op-count 0
eavt (di/init-index index datoms indexed :eavt op-count)
aevt (di/init-index index datoms indexed :aevt op-count)
avet (di/init-index index datoms indexed :avet op-count)
max-eid (init-max-eid eavt)
max-tx (get-max-tx eavt)]
max-tx (get-max-tx eavt)
op-count (count datoms)]
(map->DB (merge {:schema (merge schema (when (= :read schema-flexibility) implicit-schema))
:rschema rschema
:config config
Expand All @@ -858,6 +861,7 @@
:avet avet
:max-eid max-eid
:max-tx max-tx
:op-count op-count
:hash (hash-datoms datoms)}
(when keep-history?
{:temporal-eavt (di/empty-index index :eavt)
Expand Down Expand Up @@ -1237,30 +1241,33 @@
(let [indexing? (indexing? db (.-a datom))
a (.-a datom)
schema? (or (ds/schema-attr? a) (ds/entity-spec-attr? a))
keep-history? (and (-keep-history? db) (not (no-history? db a)))]
keep-history? (and (-keep-history? db) (not (no-history? db a)))
op-count (.op-count db)]
(if (datom-added datom)
(cond-> db
true (update-in [:eavt] #(di/-insert % datom :eavt))
true (update-in [:aevt] #(di/-insert % datom :aevt))
indexing? (update-in [:avet] #(di/-insert % datom :avet))
true (update-in [:eavt] #(di/-insert % datom :eavt op-count))
true (update-in [:aevt] #(di/-insert % datom :aevt op-count))
indexing? (update-in [:avet] #(di/-insert % datom :avet op-count))
true (advance-max-eid (.-e datom))
true (update :hash + (hash datom))
schema? (-> (update-schema datom)
update-rschema))
update-rschema)
true (update :op-count inc))
(if-some [removing ^Datom (first (-search db [(.-e datom) (.-a datom) (.-v datom)]))]
(cond-> db
true (update-in [:eavt] #(di/-remove % removing :eavt))
true (update-in [:aevt] #(di/-remove % removing :aevt))
indexing? (update-in [:avet] #(di/-remove % removing :avet))
true (update-in [:eavt] #(di/-remove % removing :eavt op-count))
true (update-in [:aevt] #(di/-remove % removing :aevt op-count))
indexing? (update-in [:avet] #(di/-remove % removing :avet op-count))
true (update :hash - (hash removing))
schema? (-> (remove-schema datom) update-rschema)
keep-history? (update-in [:temporal-eavt] #(di/-insert % removing :eavt))
keep-history? (update-in [:temporal-eavt] #(di/-insert % datom :eavt))
keep-history? (update-in [:temporal-aevt] #(di/-insert % removing :aevt))
keep-history? (update-in [:temporal-aevt] #(di/-insert % datom :aevt))
keep-history? (update-in [:temporal-eavt] #(di/-insert % removing :eavt op-count))
keep-history? (update-in [:temporal-eavt] #(di/-insert % datom :eavt (inc op-count)))
keep-history? (update-in [:temporal-aevt] #(di/-insert % removing :aevt op-count))
keep-history? (update-in [:temporal-aevt] #(di/-insert % datom :aevt (inc op-count)))
keep-history? (update :hash + (hash datom))
(and keep-history? indexing?) (update-in [:temporal-avet] #(di/-insert % removing :avet))
(and keep-history? indexing?) (update-in [:temporal-avet] #(di/-insert % datom :avet)))
(and keep-history? indexing?) (update-in [:temporal-avet] #(di/-insert % removing :avet op-count))
(and keep-history? indexing?) (update-in [:temporal-avet] #(di/-insert % datom :avet (inc op-count)))
true (update :op-count + (if (or keep-history? indexing?) 2 1)))
db))))

(defn- with-temporal-datom [db ^Datom datom]
Expand All @@ -1269,16 +1276,18 @@
current-datom ^Datom (first (-search db [(.-e datom) (.-a datom) (.-v datom)]))
history-datom ^Datom (first (search-temporal-indices db [(.-e datom) (.-a datom) (.-v datom) (.-tx datom)]))
current? (not (nil? current-datom))
history? (not (nil? history-datom))]
history? (not (nil? history-datom))
op-count (.op-count db)]
(cond-> db
current? (update-in [:eavt] #(di/-remove % current-datom :eavt))
current? (update-in [:aevt] #(di/-remove % current-datom :aevt))
(and current? indexing?) (update-in [:avet] #(di/-remove % current-datom :avet))
current? (update-in [:eavt] #(di/-remove % current-datom :eavt op-count))
current? (update-in [:aevt] #(di/-remove % current-datom :aevt op-count))
(and current? indexing?) (update-in [:avet] #(di/-remove % current-datom :avet op-count))
current? (update :hash - (hash current-datom))
(and current? schema?) (-> (remove-schema datom) update-rschema)
history? (update-in [:temporal-eavt] #(di/-remove % history-datom :eavt))
history? (update-in [:temporal-aevt] #(di/-remove % history-datom :aevt))
(and history? indexing?) (update-in [:temporal-avet] #(di/-remove % history-datom :avet)))))
history? (update-in [:temporal-eavt] #(di/-remove % history-datom :eavt op-count))
history? (update-in [:temporal-aevt] #(di/-remove % history-datom :aevt op-count))
(and history? indexing?) (update-in [:temporal-avet] #(di/-remove % history-datom :avet op-count))
(or current? history?) (update :op-count inc))))

(defn- queue-tuple [queue tuple idx db e v]
(let [tuple-value (or (get queue tuple)
Expand Down Expand Up @@ -1325,23 +1334,25 @@
(let [indexing? (indexing? db (.-a datom))
schema? (ds/schema-attr? (.-a datom))
keep-history? (and (-keep-history? db) (not (no-history? db (.-a datom)))
(not= :db/txInstant (.-a datom)))]
(not= :db/txInstant (.-a datom)))
op-count (.op-count db)]
(cond-> db
;; Optimistic removal of the schema entry (we don't know whether it is present or not)
;; Optimistic removal of the schema entry (because we don't know whether it is already present or not)
schema? (try
(-> db (remove-schema datom) update-rschema)
(catch clojure.lang.ExceptionInfo e
db))

keep-history? (update-in [:temporal-eavt] #(di/-temporal-upsert % datom :eavt))
true (update-in [:eavt] #(di/-upsert % datom :eavt))
keep-history? (update-in [:temporal-eavt] #(di/-temporal-upsert % datom :eavt op-count))
true (update-in [:eavt] #(di/-upsert % datom :eavt op-count))

keep-history? (update-in [:temporal-aevt] #(di/-temporal-upsert % datom :aevt))
true (update-in [:aevt] #(di/-upsert % datom :aevt))
keep-history? (update-in [:temporal-aevt] #(di/-temporal-upsert % datom :aevt op-count))
true (update-in [:aevt] #(di/-upsert % datom :aevt op-count))

(and keep-history? indexing?) (update-in [:temporal-avet] #(di/-temporal-upsert % datom :avet))
indexing? (update-in [:avet] #(di/-insert % datom :avet))
(and keep-history? indexing?) (update-in [:temporal-avet] #(di/-temporal-upsert % datom :avet op-count))
indexing? (update-in [:avet] #(di/-insert % datom :avet op-count))

true (update :op-count inc)
true (advance-max-eid (.-e datom))
true (update :hash + (hash datom))
schema? (-> (update-schema datom) update-rschema))))
Expand Down
Loading

0 comments on commit 287124e

Please sign in to comment.