Streaming

src/clj/jsonista/core.clj

@@ -58,15 +58,16 @@
       PersistentVectorDeserializer
       SymbolSerializer
       RatioSerializer FunctionalKeywordSerializer)
-    (com.fasterxml.jackson.core JsonGenerator$Feature JsonFactory)
+    (com.fasterxml.jackson.core JsonGenerator$Feature JsonFactory
+                                JsonParser JsonToken)
     (com.fasterxml.jackson.databind
-      JsonSerializer ObjectMapper module.SimpleModule
+      JsonSerializer ObjectMapper SequenceWriter
       SerializationFeature DeserializationFeature Module)
     (com.fasterxml.jackson.databind.module SimpleModule)
     (java.io InputStream Writer File OutputStream DataOutput Reader)
     (java.net URL)
     (com.fasterxml.jackson.datatype.jsr310 JavaTimeModule)
-    (java.util List Map Date)
+    (java.util List Map Date Iterator)
     (clojure.lang Keyword Ratio Symbol)))
 
 (defn- ^Module clojure-module
@@ -194,6 +195,76 @@
   (-read-value [this ^ObjectMapper mapper]
     (.readValue mapper this ^Class Object)))
 
+(defprotocol CreateParser
+  (-create-parser [this mapper]))
+
+(extend-protocol CreateParser
+
+  (Class/forName "[B")
+  (-create-parser [this ^ObjectMapper mapper]
+    (.createParser (.getFactory mapper) ^bytes this))
+
+  File
+  (-create-parser [this ^ObjectMapper mapper]
+    (.createParser (.getFactory mapper) this))
+
+  URL
+  (-create-parser [this ^ObjectMapper mapper]
+    (.createParser (.getFactory mapper) this))
+
+  String
+  (-create-parser [this ^ObjectMapper mapper]
+    (.createParser (.getFactory mapper) this))
+
+  Reader
+  (-create-parser [this ^ObjectMapper mapper]
+    (.createParser (.getFactory mapper) this))
+
+  InputStream
+  (-create-parser [this ^ObjectMapper mapper]
+    (.createParser (.getFactory mapper) this)))
+
+(defn ^JsonParser create-parser
+  "Create an JsonParser using given ObjectMapper.
+
+  See also: https://fasterxml.github.io/jackson-core/javadoc/2.10/com/fasterxml/jackson/core/JsonParser.html"
+  ([this]
+   (-create-parser this default-object-mapper))
+  ([this ^ObjectMapper om]
+   (-create-parser this om)))
+
+(defprotocol ReadValues
+  (-read-values [this mapper]))
+
+(extend-protocol ReadValues
+
+  (Class/forName "[B")
+  (-read-values [this ^ObjectMapper mapper]
+    (.readValues (.readerFor mapper ^Class Object) ^bytes this))
+
+  nil
+  (-read-values [_ _])
+
+  File
+  (-read-values [this ^ObjectMapper mapper]
+    (.readValues (.readerFor mapper ^Class Object) this))
+
+  URL
+  (-read-values [this ^ObjectMapper mapper]
+    (.readValues (.readerFor mapper ^Class Object) this))
+
+  String
+  (-read-values [this ^ObjectMapper mapper]
+    (.readValues (.readerFor mapper ^Class Object) this))
+
+  Reader
+  (-read-values [this ^ObjectMapper mapper]
+    (.readValues (.readerFor mapper ^Class Object) this))
+
+  InputStream
+  (-read-values [this ^ObjectMapper mapper]
+    (.readValues (.readerFor mapper ^Class Object) this)))
+
 (defprotocol WriteValue
   (-write-value [this value mapper]))
 
@@ -214,6 +285,50 @@
   (-write-value [this value ^ObjectMapper mapper]
     (.writeValue mapper this value)))
 
+(defprotocol WriteAll
+  (-write-all [this ^SequenceWriter writer]))
+
+(extend-protocol WriteAll
+
+  (Class/forName "[Ljava.lang.Object;")
+  (-write-all [this ^SequenceWriter w]
+    (.writeAll w ^"[Ljava.lang.Object;" this))
+
+  Iterable
+  (-write-all [this ^SequenceWriter w]
+    (.writeAll w this)))
+
+(defprotocol WriteValues
+  (-write-values [this values mapper]))
+
+(defmacro ^:private -write-values*
+  [this value mapper]
+  `(doto ^SequenceWriter
+       (-write-all
+        ~value
+        (-> ~mapper
+            (.writerFor Object)
+            (.without SerializationFeature/FLUSH_AFTER_WRITE_VALUE)
+            (.writeValuesAsArray ~this)))
+     (.close)))
+
+(extend-protocol WriteValues
+  File
+  (-write-values [this value ^ObjectMapper mapper]
+    (-write-values* this value mapper))
+
+  OutputStream
+  (-write-values [this value ^ObjectMapper mapper]
+    (-write-values* this value mapper))
+
+  DataOutput
+  (-write-values [this value ^ObjectMapper mapper]
+    (-write-values* this value mapper))
+
+  Writer
+  (-write-values [this value ^ObjectMapper mapper]
+    (-write-values* this value mapper)))
+
 ;;
 ;; public api
 ;;
@@ -259,3 +374,58 @@
    (-write-value to object default-object-mapper))
   ([to object ^ObjectMapper mapper]
    (-write-value to object mapper)))
+
+;; Calling iterator-seq in read-values would immediately initialize the seq.
+;; Creating the Iterable allows creating the seq, when it is needed.
+;; TODO: Name?
+(defn wrap-values
+  "Wraps Jackson Iterator into Iterable, so it can be
+  converted to a seq automatically."
+  [^Iterator iterator]
+  (when iterator
+    (reify
+      Iterable
+      (iterator [this] iterator)
+      Iterator
+      (hasNext [this] (.hasNext iterator))
+      (next [this] (.next iterator))
+      (remove [this] (.remove iterator))
+      clojure.lang.IReduceInit
+      (reduce [_ f val]
+        (loop [ret val]
+          (if (.hasNext iterator)
+            (let [ret (f ret (.next iterator))]
+              (if (reduced? ret)
+                @ret
+                (recur ret)))
+            ret)))
+      clojure.lang.Sequential)))
+
+(defn read-values
+  "Decodes a sequence of values from a JSON as an iterator
+  from anything that satisfies [[ReadValue]] protocol.
+  By default, File, URL, String, Reader and InputStream are supported.
+
+  The returned object is an Iterable, Iterator and IReduceInit.
+  It can be reduced on via [[reduce]] and turned into a lazy sequence
+  via [[iterator-seq]].
+
+  To configure, pass in an ObjectMapper created with [[object-mapper]],
+  see [[object-mapper]] docstring for the available options."
+  ([object]
+   (wrap-values (-read-values object default-object-mapper)))
+  ([object ^ObjectMapper mapper]
+   (wrap-values (-read-values object mapper))))
+
+(defn write-values
+  "Encode values as JSON and write using the provided [[WriteValue]] instance.
+  By default, File, OutputStream, DataOutput and Writer are supported.
+
+  By default, values can be an array or an Iterable.
+
+  To configure, pass in an ObjectMapper created with [[object-mapper]],
+  see [[object-mapper]] docstring for the available options."
+  ([to object]
+   (-write-values to object default-object-mapper))
+  ([to object ^ObjectMapper mapper]
+   (-write-values to object mapper)))

test/jsonista/core_test.clj

@@ -243,6 +243,40 @@
     (testing "Reader"
       (is (= original (j/read-value (InputStreamReader. (str->input-stream input-string))))))))
 
+(deftest read-values-types
+  (let [original [{"ok" 1}]
+        input-string (j/write-value-as-string original)
+        file (tmp-file)]
+    (spit file input-string)
+
+    (testing "nil"
+      (is (= nil (j/read-values nil))))
+
+    (testing "byte-array"
+      (is (= original (j/read-values (j/write-value-as-bytes original)))))
+
+    (testing "File"
+      (is (= original (j/read-values file))))
+
+    (testing "URL"
+      (is (= original (j/read-values (.toURL file)))))
+
+    (testing "String"
+      (is (= original (j/read-values input-string))))
+
+    (testing "InputStream"
+      (is (= original (j/read-values (str->input-stream input-string)))))
+
+    (testing "Reader"
+      (is (= original (j/read-values (InputStreamReader. (str->input-stream input-string))))))
+
+    (testing "JsonParser"
+      (let [parser (j/create-parser input-string)]
+        ;; token = nil, start of document
+        (.nextToken parser) ;; START_ARRAY
+        (.nextToken parser) ;; START_OBJECT
+        (is (= original (vec (j/wrap-values (.readValuesAs parser ^Class Object)))))))))
+
 (deftest write-value-types
   (let [original {"ok" 1}
         expected (j/write-value-as-string original)
@@ -268,3 +302,71 @@
       (is (= expected (slurp file)))
       (.delete file))))
 
+(deftest write-values-types
+  (let [original [{"ok" 1}]
+        expected (j/write-value-as-string original)
+        file (tmp-file)]
+
+    (testing "File"
+      (j/write-values file original)
+      (is (= expected (slurp file)))
+      (.delete file))
+
+    (testing "OutputStream"
+      (j/write-values (FileOutputStream. file) original)
+      (is (= expected (slurp file)))
+      (.delete file))
+
+    (testing "DataOutput"
+      (j/write-values (RandomAccessFile. file "rw") original)
+      (is (= expected (slurp file)))
+      (.delete file))
+
+    (testing "Writer"
+      (j/write-values (FileWriter. file) original)
+      (is (= expected (slurp file)))
+      (.delete file))))
+
+(deftest read-values-iteration
+  (let [original [{"ok" 1}]
+        ^java.util.Iterator it (j/read-values (j/write-value-as-bytes original))]
+    (is (instance? java.util.Iterator it))
+    (is (.hasNext it))
+    (is (= (first original) (.next it)))
+    (is (false? (.hasNext it)))))
+
+(deftest read-values-reduction
+  (let [original [{"ok" 1}]
+        ^java.util.Iterator it (j/read-values (j/write-value-as-bytes original))
+        xf (map #(update % "ok" inc))]
+    (is (= (into [] xf original) (into [] xf it)))))
+
+(deftest write-values-iterable
+  (let [original [{"ok" 1}]
+        xf (map #(update % "ok" inc))
+        expected (j/write-value-as-string (into [] xf original))
+        file (tmp-file)
+        eduction (->Eduction xf original)]
+
+    (j/write-values file eduction)
+    (is (= expected (slurp file)))
+    (.delete file)))
+
+(deftest read-values-parser-example
+  (let [original {"type" "FeatureCollection"
+                  "features" [{"value" 1}
+                              {"value" 1}
+                              {"value" 1}]
+                  "foo" "bar"}
+        input-string (j/write-value-as-string original)]
+    (testing "JsonParser"
+      (let [parser (j/create-parser input-string)]
+        ;; token = nil, start of document
+        (.nextToken parser) ;; START_OBJECT
+        (.nextToken parser) ;; "type"
+        (.nextToken parser) ;; "FeatureCollection"
+        (.nextToken parser) ;; "features"
+        (.nextToken parser) ;; START_ARRAY
+        (.nextToken parser) ;; START_OBJECT (the first item)
+        (is (= (get original "features")
+               (iterator-seq (.readValuesAs parser ^Class Object))))))))